Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
import { createHash } from 'node:crypto';
import { chmod, mkdir, mkdtemp, readdir, readFile, rm, symlink, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { Client } from '@modelcontextprotocol/sdk/client/index.js';
import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js';
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest';
import type { HttpMcpServerSpec } from '../adapter';
import { Redactions } from './redaction';
import type { CliTool, ServiceGrant, ServiceTokenAnswer } from './service-access';
import { ServiceCredential } from './service-credential';
import type { HostAsk } from './session-channel';
import {
checkCommand,
CommandRefused,
refusedToken,
runEnvironment,
startVendorClis,
} from './vendor-cli';
const GOOD = 'synthetic-cli-token-the-vendor-takes';
const STALE = 'stale-synthetic-cli-token-the-vendor-refuses';
/**
* A stand-in for a vendor's tool, never the real one: it answers by its
* second argument, refuses a token that starts with `stale`, and says what it
* was given, its token only as a hash unless asked to leak it.
*/
const FAKE = `
import { createHash } from 'node:crypto';
import { existsSync, readFileSync, statSync, writeFileSync } from 'node:fs';
import { basename } from 'node:path';
const [, , , what, ...rest] = process.argv;
const token = process.env.EXCLI_TOKEN ?? '';
if (token.startsWith('stale')) {
console.log(JSON.stringify({ error: { code: 401, message: 'Invalid credentials' } }));
process.exit(1);
}
const after = (flag) => rest[rest.indexOf(flag) + 1];
switch (what) {
case 'env':
console.log(JSON.stringify({
keys: Object.keys(process.env).sort(),
token: createHash('sha256').update(token).digest('hex'),
config: process.env.EXCLI_CONFIG_DIR,
home: process.env.HOME,
cwd: process.cwd(),
dotenv: existsSync('.env') ? statSync('.env').size : null,
}));
break;
case 'argv':
console.log(JSON.stringify(process.argv.slice(2)));
break;
case 'key':
console.log(['api', process.env.EXCLI_API_KEY, 'app', process.env.EXCLI_APP_KEY, 'site', process.env.EXCLI_SITE, 'token', process.env.EXCLI_TOKEN ?? 'none'].join(' '));
console.error('stderr ' + process.env.EXCLI_APP_KEY);
process.exit(2);
case 'leak':
console.log('stdout ' + token);
console.error('stderr ' + token);
process.exit(3);
case 'big':
process.stdout.write('x'.repeat(Number(rest[0])));
break;
case 'sleep':
setTimeout(() => console.log('woke'), Number(rest[0]));
break;
case 'write': {
const flag = rest.find((arg) => arg.startsWith('-o') || arg.startsWith('--output'));
const path = flag === '--output' || flag === '-o' ? after(flag) : flag.replace(/^(--output=|-o=?)/, '');
writeFileSync(path, 'report from the vendor');
console.log('saved ' + path);
break;
}
case 'read': {
const path = after('--upload');
console.log(basename(path) + ' ' + readFileSync(path, 'utf8'));
break;
}
case '+put':
console.log(basename(rest[0]) + ' ' + readFileSync(rest[0], 'utf8'));
break;
case 'download':
writeFileSync('download.pdf', '%PDF');
console.log(JSON.stringify({ saved_file: 'download.pdf' }));
break;
case 'fail':
console.log(JSON.stringify({ error: { code: 403, message: 'Forbidden' } }));
process.exit(1);
default:
console.log('ok ' + what);
}
`;
const TOOL: CliTool = {
name: 'example-cli',
binary: 'excli',
token_env: 'EXCLI_TOKEN',
env: null,
config_env: 'EXCLI_CONFIG_DIR',
allow: ['items', 'boards'],
deny: ['auth', '--profile'],
path_flags: { '--upload': 'read', '--output': 'write', '-o': 'write' },
path_args: [{ after: '+put', direction: 'read' }],
output_cap_bytes: 2048,
timeout_s: 30,
token_refused: { exit_code: 1, json_path: 'error.code', value: 401 },
release: { version: '1.2.3', targets: {} },
};
/** The tool as an api_key entry has it: the key's fields in variables of their own. */
const KEY_TOOL: CliTool = {
...TOOL,
token_env: null,
env: {
EXCLI_API_KEY: '{api_key}',
EXCLI_APP_KEY: '{application_key}',
EXCLI_SITE: 'https://api.{site}',
},
};
const API_KEY = 'synthetic-api-key-for-the-cli';
const APP_KEY = 'synthetic-application-key-for-the-cli';
const KEY = JSON.stringify({
public: { site: 'datadoghq.eu' },
secret: { api_key: API_KEY, application_key: APP_KEY },
});
function grant(tool: CliTool = TOOL): ServiceGrant {
return {
service: 'example',
access: 'write',
tool_mode: 'deny',
tools: [],
resources: {},
skill: null,
mcp_servers: [],
cli_tools: [tool],
};
}
const sha = (value: string) => createHash('sha256').update(value).digest('hex');
/** Variables a POSIX shell sets for itself, which the test's wrapper adds. */
const SHELL_SETS = ['PWD', 'OLDPWD', 'SHLVL', '_', '__CF_USER_TEXT_ENCODING'];
describe('checkCommand', () => {
const refused = (args: unknown) => () => checkCommand(TOOL, args);
it('takes an allowed command and finds its file arguments in every form', () => {
const checked = checkCommand(TOOL, [
'items',
'get',
'--output',
'a.bin',
'--output=b.bin',
'-o',
'c.bin',
'-o=d.bin',
'-oe.bin',
'--upload',
'in.txt',
'--params',
'{"q":"--output"}',
]);
expect(checked.paths.map(({ prefix, path, direction }) => [prefix, path, direction])).toEqual([
['', 'a.bin', 'write'],
['--output=', 'b.bin', 'write'],
['', 'c.bin', 'write'],
['-o=', 'd.bin', 'write'],
['-o', 'e.bin', 'write'],
['', 'in.txt', 'read'],
]);
});
it.each([
[['auth', 'login'], 'not available'],
[['gmail', 'users'], 'not available'],
[['items:v2', 'list'], 'not available'],
[['--profile', 'x', 'items'], 'not available'],
[['items', '--profile=work'], '`--profile` is not available'],
[['items', '--profile', 'work'], '`--profile` is not available'],
[['items', '-ho', 'x.bin'], 'may not be combined'],
[['items', '--output'], 'needs a file path'],
[['items', '--output='], 'needs a file path'],
[['items', '--', '--output', 'x'], '`--` is not accepted'],
[[], 'Give a command'],
['items list', 'list of strings'],
[['items', 'a\0b'], 'NUL'],
])('refuses %j', (args, message) => {
expect(refused(args)).toThrow(CommandRefused);
expect(refused(args)).toThrow(message);
});
it('finds a positional file argument after its word, wherever the word stands', () => {
expect(checkCommand(TOOL, ['items', '+put', 'a.txt', '--name', 'A']).paths).toEqual([
{ index: 2, prefix: '', path: 'a.txt', direction: 'read' },
]);
expect(checkCommand(TOOL, ['items', '--format', 'json', '+put', '/etc/hosts']).paths).toEqual([
{ index: 4, prefix: '', path: '/etc/hosts', direction: 'read' },
]);
});
it.each([[['items', '+put']], [['items', '+put', '--name', 'x', 'a.txt']]])(
'refuses a positional file argument that is not right after its word: %j',
(args) => {
expect(refused(args)).toThrow('takes a file path right after it');
}
);
it('lets a negative number through as a value', () => {
expect(checkCommand(TOOL, ['items', 'list', '--offset', '-10']).paths).toEqual([]);
});
});
describe('refusedToken', () => {
it('reads the catalog’s exit code and JSON value, and nothing else', () => {
expect(refusedToken(TOOL, 1, '{"error":{"code":401}}')).toBe(true);
expect(refusedToken(TOOL, 1, '{"error":{"code":403}}')).toBe(false);
expect(refusedToken(TOOL, 2, '{"error":{"code":401}}')).toBe(false);
expect(refusedToken(TOOL, 1, 'error 401')).toBe(false);
});
});
describe('runEnvironment', () => {
it('holds the token, the folders and the host’s network settings, and nothing else', () => {
const env = runEnvironment({
tool: TOOL,
credential: ServiceCredential.of(GOOD),
home: '/h',
config: '/c',
tmp: '/t',
hostEnv: {
PATH: '/usr/bin',
HTTPS_PROXY: 'http://proxy.example.test:3128',
SSL_CERT_FILE: '/etc/ca.pem',
SWITCH_API_TOKEN: 'switch-secret',
GH_TOKEN: 'other-secret',
},
platform: 'linux',
});
expect(env).toEqual({
HTTPS_PROXY: 'http://proxy.example.test:3128',
SSL_CERT_FILE: '/etc/ca.pem',
HOME: '/h',
TMPDIR: '/t',
EXCLI_CONFIG_DIR: '/c',
EXCLI_TOKEN: GOOD,
});
});
it('sets each of a key’s variables from its template, and no token variable', () => {
const env = runEnvironment({
tool: KEY_TOOL,
credential: ServiceCredential.of(KEY),
home: '/h',
config: '/c',
tmp: '/t',
hostEnv: {},
platform: 'linux',
});
expect(env).toEqual({
HOME: '/h',
TMPDIR: '/t',
EXCLI_CONFIG_DIR: '/c',
EXCLI_API_KEY: API_KEY,
EXCLI_APP_KEY: APP_KEY,
EXCLI_SITE: 'https://api.datadoghq.eu',
});
});
it('refuses a template naming a field the key does not carry', () => {
expect(() =>
runEnvironment({
tool: { ...KEY_TOOL, env: { EXCLI_REGION: '{region}' } },
credential: ServiceCredential.of(KEY),
home: '/h',
config: '/c',
tmp: '/t',
hostEnv: {},
platform: 'linux',
})
).toThrow('names {region}');
});
});
describe.skipIf(process.platform === 'win32')('startVendorClis', () => {
let root: string;
let binary: string;
const closers: (() => Promise<void>)[] = [];
beforeAll(async () => {
root = await mkdtemp(join(tmpdir(), 'vendor-cli-'));
const fake = join(root, 'fake-excli.mjs');
await writeFile(fake, FAKE);
binary = join(root, 'excli');
// Run directly, as the real binary would be: the wrapper only hands
// every argument on as it came.
await writeFile(binary, `#!/bin/sh\nexec "${process.execPath}" "${fake}" "$@"\n`);
await chmod(binary, 0o755);
});
afterAll(async () => {
await rm(root, { recursive: true, force: true });
});
let session: string;
let state: string;
beforeEach(async () => {
session = await mkdtemp(join(root, 'session-'));
state = await mkdtemp(join(root, 'state-'));
});
afterEach(async () => {
for (const close of closers.splice(0)) await close();
});
async function started(answers: ServiceTokenAnswer[], tool: CliTool = TOOL) {
const asked: HostAsk[] = [];
const redactions = new Redactions();
const servers = await startVendorClis({
grants: [grant(tool)],
ask: async (ask) => {
asked.push(ask);
const next = answers.shift();
if (!next) throw new Error('no answer left');
return next;
},
redactions,
cwd: session,
stateDir: state,
binary: async () => binary,
hostEnv: { PATH: process.env.PATH, HTTPS_PROXY: 'http://proxy.example.test:3128' },
platform: process.platform,
});
closers.push(servers.close);
const spec = servers.specs['example-cli'] as HttpMcpServerSpec;
const transport = new StreamableHTTPClientTransport(new URL(spec.url), {
requestInit: { headers: spec.headers },
});
const client = new Client({ name: 'coding-tool', version: '1.0.0' });
await client.connect(transport);
closers.push(() => client.close());
const run = async (args: unknown) => {
const result = (await client.callTool({ name: 'excli', arguments: { args } })) as {
isError?: boolean;
content: { type: string; text: string }[];
};
return { isError: result.isError ?? false, text: result.content[0].text };
};
return { asked, redactions, servers, client, run, spec };
}
const token = (value: string): ServiceTokenAnswer => ({
kind: 'token',
token: value,
expiresAt: new Date(Date.now() + 3_600_000).toISOString(),
});
it('offers one tool, behind a bearer the coding tool is given, with no token in sight', async () => {
const { client, spec, asked } = await started([]);
const { tools } = await client.listTools();
expect(tools.map((tool) => tool.name)).toEqual(['excli']);
expect(tools[0].description).toContain('items, boards');
expect(JSON.stringify(spec)).not.toContain(GOOD);
expect(asked).toEqual([]);
});
it('runs in a folder of its own with only its token, folders and the host’s network settings', async () => {
const { run, asked, redactions } = await started([token(GOOD)]);
const result = await run(['items', 'env']);
expect(result.isError).toBe(false);
const seen = JSON.parse(result.text);
expect(seen.token).toBe(sha(GOOD));
// Less what the test's /bin/sh wrapper sets for itself.
expect(seen.keys.filter((key: string) => !SHELL_SETS.includes(key))).toEqual(
['EXCLI_CONFIG_DIR', 'EXCLI_TOKEN', 'HOME', 'HTTPS_PROXY', 'TMPDIR'].sort()
);
expect(seen.cwd).not.toBe(session);
expect(seen.cwd.startsWith(state) || seen.cwd.includes('/runs/run-')).toBe(true);
expect(seen.dotenv).toBe(0);
expect(seen.config).toBe(join(state, 'example-cli', 'config'));
expect(asked).toEqual([{ type: 'service-token', service: 'example', rejected: null }]);
expect(redactions.list()).toContain(GOOD);
// The run's own folder is gone once it is answered.
expect(await readdir(join(state, 'example-cli', 'runs'))).toEqual([]);
});
it('gives shell metacharacters to the tool as plain arguments', async () => {
const { run } = await started([token(GOOD)]);
const args = ['items', 'argv', '$(touch pwned)', '; rm -rf .', '`id`', 'a\'b"c', '*'];
const result = await run(args);
expect(JSON.parse(result.text)).toEqual(args);
expect(await readdir(session)).toEqual([]);
});
it('never passes the token on, even when the tool prints it', async () => {
const { run } = await started([token(GOOD)]);
const result = await run(['items', 'leak']);
expect(result.isError).toBe(true);
expect(result.text).toContain('exited with code 3');
expect(result.text).not.toContain(GOOD);
expect(result.text).toContain('stdout [REDACTED]');
expect(result.text).toContain('stderr [REDACTED]');
});
it('runs with a key’s fields in their variables, and scrubs each secret from the output', async () => {
const { run, redactions } = await started([token(KEY)], KEY_TOOL);
const result = await run(['items', 'key']);
expect(result.isError).toBe(true);
expect(result.text).toContain(
'api [REDACTED] app [REDACTED] site https://api.datadoghq.eu token none'
);
expect(result.text).toContain('stderr [REDACTED]');
expect(result.text).not.toContain(API_KEY);
expect(result.text).not.toContain(APP_KEY);
expect(redactions.list()).toEqual(expect.arrayContaining([KEY, API_KEY, APP_KEY]));
expect(redactions.list()).not.toContain('datadoghq.eu');
});
it('asks again once when the vendor refuses the token, naming the refused one', async () => {
const { run, asked } = await started([token(STALE), token(GOOD)]);
const result = await run(['items', 'hello']);
expect(result).toEqual({ isError: false, text: 'ok hello\n' });
expect(asked.map((ask) => (ask.type === 'service-token' ? ask.rejected : null))).toEqual([
null,
STALE,
]);
});
it('stops after a second refusal, saying so', async () => {
const { run, asked } = await started([token(STALE), token(STALE)]);
const result = await run(['items', 'hello']);
expect(result.isError).toBe(true);
expect(result.text).toContain('refused this agent’s token again'.replace('’', "'"));
expect(asked).toHaveLength(2);
});
it('passes on a refusal from Switch', async () => {
const { run } = await started([
{ kind: 'refused', code: 'grant_missing', message: 'This agent has no grant.', final: true },
]);
const result = await run(['items', 'hello']);
expect(result.isError).toBe(true);
expect(result.text).toContain('This agent has no grant.');
});
it('answers a vendor error as an error, without asking again', async () => {
const { run, asked } = await started([token(GOOD)]);
const result = await run(['items', 'fail']);
expect(result.isError).toBe(true);
expect(result.text).toContain('"code":403');
expect(asked).toHaveLength(1);
});
it('refuses a command before asking for a token', async () => {
const { run, asked } = await started([]);
const result = await run(['auth', 'login']);
expect(result.isError).toBe(true);
expect(result.text).toContain('not available');
expect(asked).toEqual([]);
});
it('stops a run that outlives its timeout', async () => {
const { run } = await started([token(GOOD)], { ...TOOL, timeout_s: 1 });
const started_ = Date.now();
const result = await run(['items', 'sleep', '10000']);
expect(result.isError).toBe(true);
expect(result.text).toContain('did not finish within 1 s');
expect(Date.now() - started_).toBeLessThan(8_000);
});
it('saves output past the cap to a file git ignores, and returns its path', async () => {
const { run } = await started([token(GOOD)]);
const result = await run(['items', 'big', '5000']);
expect(result.isError).toBe(false);
const path = /saved at (\S+)\./.exec(result.text)?.[1];
expect(path).toMatch(/^\.switch\/example-cli\/output-/);
expect(await readFile(join(session, path!), 'utf8')).toBe('x'.repeat(5000));
expect(await readFile(join(session, '.switch', 'example-cli', '.gitignore'), 'utf8')).toBe(
'*\n'
);
expect(Buffer.byteLength(result.text)).toBeLessThan(5000);
});
it.each([
[['--output', 'out/report.txt']],
[['--output=out/report.txt']],
[['-o', 'out/report.txt']],
[['-oout/report.txt']],
])('writes a file the command names into the session’s folder: %j', async (flag) => {
await mkdir(join(session, 'out'));
const { run } = await started([token(GOOD)]);
const result = await run(['items', 'write', ...flag]);
expect(result.isError).toBe(false);
expect(result.text).toMatch(/Wrote (out\/report.txt)\./);
expect(await readFile(join(session, 'out', 'report.txt'), 'utf8')).toBe(
'report from the vendor'
);
});
it('reads a file the command names from the session’s folder', async () => {
await writeFile(join(session, 'notes.md'), 'hello drive');
const { run } = await started([token(GOOD)]);
const result = await run(['items', 'read', '--upload', 'notes.md']);
expect(result).toEqual({ isError: false, text: 'notes.md hello drive\n' });
});
it.each([
[['--upload', '../outside.txt'], 'outside this session'],
[['--upload', '/etc/hosts'], 'outside this session'],
[['--upload', 'missing.txt'], 'There is no file'],
[['--output', '../outside.txt'], 'outside this session'],
[['--output', 'no-such-folder/x.txt'], 'does not exist'],
[['--output', 'out/'], 'names a folder'],
[['--upload', 'escape'], 'outside this session'],
])('refuses a path outside the session’s folder: %j', async (flag, message) => {
await mkdir(join(session, 'out'));
await writeFile(join(root, 'outside.txt'), 'secret');
await symlink(join(root, 'outside.txt'), join(session, 'escape'));
const { run, asked } = await started([token(GOOD)]);
const result = await run(['items', 'read', ...flag]);
expect(result.isError).toBe(true);
expect(result.text).toContain(message);
expect(result.text).not.toContain('secret');
expect(asked).toEqual([]);
});
it('reads a positional file from the session’s folder, and nothing outside it', async () => {
await writeFile(join(session, 'deck.md'), 'slides');
const { run } = await started([token(GOOD)]);
expect(await run(['items', '+put', 'deck.md'])).toEqual({
isError: false,
text: 'deck.md slides\n',
});
const outside = await run(['items', '--format', 'json', '+put', '/etc/hosts']);
expect(outside.isError).toBe(true);
expect(outside.text).toContain('outside this session');
});
it('keeps what a run leaves behind unasked, and says where', async () => {
const { run } = await started([token(GOOD)]);
const result = await run(['items', 'download']);
const saved = /Saved (\S+)\./.exec(result.text)?.[1];
expect(saved).toMatch(/^\.switch\/example-cli\/\d+-download\.pdf$/);
expect(await readFile(join(session, saved!), 'utf8')).toBe('%PDF');
});
it('leaves no token in any file, the session’s or its own', async () => {
const { run } = await started([token(GOOD), token(GOOD), token(GOOD)]);
await run(['items', 'env']);
await run(['items', 'big', '5000']);
await run(['items', 'leak']);
for (const folder of [session, state])
for (const entry of await readdir(folder, { recursive: true, withFileTypes: true })) {
if (!entry.isFile()) continue;
const content = await readFile(join(entry.parentPath, entry.name), 'utf8');
expect(content).not.toContain(GOOD);
}
});
});