A team of long-lived bots for DeepSeek Harness, with main bots, bot-to-bot messages and group chats.
Contain connector gateway stream failures and log HTTP errors
Fei2411ef4763da5f5c
diff --git a/ds-bot/src/host/connectors/gateway.js b/ds-bot/src/host/connectors/gateway.jsindex 11213b1f..6265830b 100644--- a/ds-bot/src/host/connectors/gateway.js+++ b/ds-bot/src/host/connectors/gateway.js@@ -36,6 +36,7 @@ const upstreamRequest = (target, init, onResponse) =>(target.protocol === 'http:' && loopbackHttp(target) ? httpRequest : httpsRequest)(target, init, onResponse)const readAll = stream => new Promise((resolve, reject) => {+ if (stream.errored) { reject(stream.errored); return }const chunks = []stream.on('data', chunk => chunks.push(chunk))stream.on('end', () => resolve(Buffer.concat(chunks)))@@ -71,17 +72,24 @@ export function createGateway({ credential, readOnly, onUnauthorized, log = () =error: { code: -32600, message: `The ${id} connector is read-only: this tool can change things, so it is off. Turn off the read-only switch in Settings → Connectors to use it.` },})- async function sendUpstream(entry, method, path, headers, body) {+ async function sendUpstream(id, entry, method, path, headers, body) {return new Promise((resolve, reject) => {const target = new URL(entry.url)- if (path !== '/' && path !== '') {- target.pathname = entry.pathBase + path.replace(/^\//, '')+ const suffix = new URL(path, 'http://x')+ if (suffix.pathname !== '/') {+ target.pathname = `${target.pathname.replace(/\/$/, '')}/${suffix.pathname.replace(/^\//, '')}`}- for (const [key, value] of new URL(path, 'http://x').searchParams) target.searchParams.append(key, value)+ for (const [key, value] of suffix.searchParams) target.searchParams.append(key, value)const outgoing = { ...entry.headers }for (const name of FORWARD_HEADERS) if (headers[name] !== undefined) outgoing[name] = headers[name]if (body !== undefined) outgoing['content-length'] = body.length- const req = upstreamRequest(target, { method, headers: outgoing }, resolve)+ const req = upstreamRequest(target, { method, headers: outgoing }, upstream => {+ // This listener must exist before refresh awaits: the response can fail+ // before the handler starts consuming it.+ upstream.on('error', reject)+ if (upstream.statusCode >= 400) log('ds-bot: connector %s upstream %s HTTP %s (session: %s)', id, method, upstream.statusCode, headers['mcp-session-id'] !== undefined)+ resolve(upstream)+ })req.on('error', reject)if (body !== undefined) req.write(body)req.end()@@ -102,7 +110,6 @@ export function createGateway({ credential, readOnly, onUnauthorized, log = () =}const suffix = req.url.slice(`/c/${id}`.length)const path = suffix === '' ? '/' : suffix- entry.pathBase = new URL(entry.url).pathname.replace(/\/$/, '') === '' ? '' : `${new URL(entry.url).pathname.replace(/\/$/, '')}/`const limited = readOnly(id) === trueconst body = req.method === 'GET' || req.method === 'HEAD' ? undefined : await readAll(req)// Read-only: a write tools/call never reaches the server.@@ -121,12 +128,12 @@ export function createGateway({ credential, readOnly, onUnauthorized, log = () =return}if (rpc?.method === 'tools/list') {- let upstream = await sendUpstream(entry, req.method, path, req.headers, body)+ let upstream = await sendUpstream(id, entry, req.method, path, req.headers, body)if (upstream.statusCode === 401) {const retried = await onUnauthorized(id)if (retried === true) {upstream.resume()- upstream = await sendUpstream(credential(id) ?? entry, req.method, path, req.headers, body)+ upstream = await sendUpstream(id, credential(id) ?? entry, req.method, path, req.headers, body)}}const raw = await readAll(upstream)@@ -156,15 +163,16 @@ export function createGateway({ credential, readOnly, onUnauthorized, log = () =}let upstreamtry {- upstream = await sendUpstream(entry, req.method, path, req.headers, body)+ upstream = await sendUpstream(id, entry, req.method, path, req.headers, body)// One refresh + retry when the answer is still unsent.if (upstream.statusCode === 401) {const retried = await onUnauthorized(id)if (retried === true) {upstream.resume()- upstream = await sendUpstream(credential(id) ?? entry, req.method, path, req.headers, body)+ upstream = await sendUpstream(id, credential(id) ?? entry, req.method, path, req.headers, body)}}+ if (upstream.errored) throw upstream.errored} catch (error) {log('ds-bot: connector gateway %s: %s', id, error?.message ?? error)res.writeHead(502, { 'content-type': 'application/json' })@@ -179,6 +187,10 @@ export function createGateway({ credential, readOnly, onUnauthorized, log = () =const close = () => { upstream.destroy() }res.on('close', close)upstream.on('end', () => res.off('close', close))+ upstream.on('error', () => {+ log('ds-bot: connector %s upstream %s stream failed', id, req.method)+ res.destroy()+ })res.writeHead(upstream.statusCode ?? 502, heads)upstream.pipe(res)return@@ -203,6 +215,8 @@ export function createGateway({ credential, readOnly, onUnauthorized, log = () =async listen() {if (server !== null) returnserver = createServer((req, res) => {+ req.on('error', () => res.destroy())+ res.on('error', () => req.destroy())const found = /^\/c\/([A-Za-z0-9_-]{1,32})/.exec(req.url ?? '')if (found === null || !nonceMatches(req.headers.authorization, nonce)) {res.writeHead(404)@@ -211,10 +225,11 @@ export function createGateway({ credential, readOnly, onUnauthorized, log = () =}handle(found[1], req, res).catch((error) => {log('ds-bot: connector gateway: %s', error?.message ?? error)- if (!res.headersSent) res.writeHead(500)- res.end()+ if (res.headersSent) res.destroy()+ else { res.writeHead(502); res.end() }})})+ server.on('error', error => log('ds-bot: connector gateway server failed: %s', error?.code ?? 'error'))await new Promise((resolve, reject) => {server.once('error', reject)server.listen(0, '127.0.0.1', resolve)diff --git a/ds-bot/test/connectors-engine.test.js b/ds-bot/test/connectors-engine.test.jsindex 01b1af93..463065b0 100644--- a/ds-bot/test/connectors-engine.test.js+++ b/ds-bot/test/connectors-engine.test.js@@ -685,7 +685,7 @@ test('gateway keeps the entry path and merges queries', async (t) => {await fetch(gw.urlFor('svc'), { method: 'POST', headers: head, body: '{}' })assert.equal(seen.at(-1), '/mcp?key=K1')await fetch(`${gw.urlFor('svc')}?x=1&y=2`, { method: 'POST', headers: head, body: '{}' })- assert.match(seen.at(-1), /^\/mcp\?key=K1&x=1&y=2$|x=1.*key=K1/)+ assert.equal(seen.at(-1), '/mcp?key=K1&x=1&y=2')})test('gateway destroys the upstream stream when the client disconnects', async (t) => {diff --git a/ds-bot/test/connectors-gateway.test.js b/ds-bot/test/connectors-gateway.test.jsnew file mode 100644index 00000000..a9f0aa0b--- /dev/null+++ b/ds-bot/test/connectors-gateway.test.js@@ -0,0 +1,84 @@+import assert from 'node:assert/strict'+import { spawnSync } from 'node:child_process'+import { createServer } from 'node:http'+import { test } from 'node:test'+import { createGateway } from '../src/host/connectors/gateway.js'++for (const refreshing of [false, true]) {+ test(`an upstream stream error cannot crash the engine${refreshing ? ' during refresh' : ''}`, () => {+ const script = `+ import assert from 'node:assert/strict'+ import { createServer } from 'node:http'+ import { createGateway } from ${JSON.stringify(new URL('../src/host/connectors/gateway.js', import.meta.url).href)}+ const upstream = createServer((req, res) => {+ res.writeHead(${refreshing ? 401 : 200}, { 'content-type': 'text/event-stream' })+ res.write('data: {}\\n\\n')+ setTimeout(() => res.destroy(), 10)+ })+ await new Promise(resolve => upstream.listen(0, '127.0.0.1', resolve))+ const gw = createGateway({+ credential: () => ({ url: 'http://127.0.0.1:' + upstream.address().port, headers: {} }),+ readOnly: () => false,+ onUnauthorized: async () => { await new Promise(resolve => setTimeout(resolve, 30)); return false },+ })+ await gw.listen()+ try {+ const response = await fetch(gw.urlFor('svc'), { headers: gw.headers(), signal: AbortSignal.timeout(1000) })+ ${refreshing ? 'assert.equal(response.status, 502)' : ''}+ ${refreshing ? 'await response.text()' : "await assert.rejects(response.text(), error => error.name !== 'TimeoutError' && error.name !== 'AbortError')"}+ } finally { await gw.close(); await new Promise(resolve => upstream.close(resolve)) }+ `+ const result = spawnSync(process.execPath, ['--input-type=module', '-e', script], { encoding: 'utf8', timeout: 5000 })+ assert.equal(result.status, 0, result.stderr || result.error?.message)+ })+}++test('gateway preserves MCP POST headers and bodies and reports expired sessions without replaying calls', async (t) => {+ const seen = []+ const upstream = createServer((req, res) => {+ const chunks = []+ req.on('data', chunk => chunks.push(chunk))+ req.on('end', () => {+ seen.push({ headers: req.headers, body: Buffer.concat(chunks).toString(), url: req.url })+ res.writeHead(seen.length === 1 ? 200 : 404, {+ 'content-type': seen.length === 1 ? 'text/event-stream' : 'application/json',+ 'mcp-session-id': 'private-session',+ })+ res.end(seen.length === 1 ? 'event: message\ndata: {"jsonrpc":"2.0","id":1,"result":{}}\n\n' : '{"error":"Session not found"}')+ })+ })+ await new Promise(resolve => upstream.listen(0, '127.0.0.1', resolve))+ t.after(() => upstream.close())+ const logs = []+ let refreshes = 0+ const gw = createGateway({+ credential: () => ({ url: `http://127.0.0.1:${upstream.address().port}/mcp?key=private-key`, headers: { Authorization: 'Bearer private-token' } }),+ readOnly: () => false,+ onUnauthorized: async () => { refreshes += 1; return false },+ log: (...args) => logs.push(args),+ })+ await gw.listen()+ t.after(() => gw.close())+ const headers = { ...gw.headers(), accept: 'application/json, text/event-stream', 'content-type': 'application/json', 'mcp-protocol-version': '2025-03-26' }+ const body = '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}'+ const response = await fetch(gw.urlFor('notion'), { method: 'POST', headers, body })+ assert.equal(response.status, 200)+ assert.equal(response.headers.get('mcp-session-id'), 'private-session')+ assert.equal(response.headers.get('content-type'), 'text/event-stream')+ assert.match(await response.text(), /event: message/)+ assert.equal(seen[0].body, body)+ assert.equal(seen[0].headers.accept, headers.accept)+ assert.equal(seen[0].headers['mcp-protocol-version'], headers['mcp-protocol-version'])+ assert.equal(seen[0].headers.authorization, 'Bearer private-token')+ const expired = await fetch(gw.urlFor('notion'), {+ method: 'POST', headers: { ...headers, 'mcp-session-id': 'private-session' },+ body: '{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"write"}}',+ })+ assert.equal(expired.status, 404)+ assert.equal((await expired.json()).error, 'Session not found')+ assert.equal(seen[1].headers['mcp-session-id'], 'private-session')+ assert.equal(seen.length, 2)+ assert.equal(refreshes, 0)+ assert.match(JSON.stringify(logs), /notion.*POST.*404/)+ assert.equal(JSON.stringify(logs).includes('private-'), false)+})