Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
Make the Console tests that failed intermittently in CI deterministic
Petr Bauch273735f5e638
diff --git a/console/apps/switch-console-desktop/src/main/core/agents/remote-dir.integration.test.ts b/console/apps/switch-console-desktop/src/main/core/agents/remote-dir.integration.test.tsindex f20bf64f2..12a4455da 100644--- a/console/apps/switch-console-desktop/src/main/core/agents/remote-dir.integration.test.ts+++ b/console/apps/switch-console-desktop/src/main/core/agents/remote-dir.integration.test.ts@@ -44,7 +44,9 @@ vi.mock('@main/core/locations/location-transport', () => ({const { inspectRemoteDir } = await import('./remote-dir');beforeAll(async () => {- const { private: hostKey } = utils.generateKeyPairSync('ed25519');+ // Not ed25519: ssh2's ed25519 generator strips a leading zero byte from the+ // public key, so about one key in 256 is malformed. ECDSA keys always parse.+ const { private: hostKey } = utils.generateKeyPairSync('ecdsa', { bits: 256 });server = new Server({ hostKeys: [hostKey] }, (conn) => {conn.on('authentication', (ctx) => ctx.accept());diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/stack-lock.docker.test.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/stack-lock.docker.test.tsindex 12d5132a1..0a899af54 100644--- a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/stack-lock.docker.test.ts+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/stack-lock.docker.test.ts@@ -89,16 +89,64 @@ const bob = (overrides: Partial<LockClaim> = {}) =>...overrides,});-/** No renewals unless a test asks for them, so expiry can be watched. */-const QUIET: LockTiming = {- ttlSeconds: 2,+/**+ * A lease that outlasts any test, for tests that need the lock to stay held+ * while they act on it. Each step here starts a container, so a short lease+ * can lapse between two of them on a slow machine.+ */+const HELD: LockTiming = {+ ttlSeconds: 600,renewEveryMs: 60_000,maxHoldSeconds: 600,pollEveryMs: 200,};+/** No renewals unless a test asks for them, so expiry can be watched. */+const LAPSING: LockTiming = { ...HELD, ttlSeconds: 2 };+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));+/** `promise`, and whether it has settled yet. */+function settles<T>(promise: Promise<T>): { promise: Promise<T>; settled: () => boolean } {+ let settled = false;+ const tracked = promise.finally(() => {+ settled = true;+ });+ return { promise: tracked, settled: () => settled };+}++/**+ * Holds the state mutex from a container of its own until released, so a test+ * can show a script waiting for it however long containers take to start.+ */+async function holdStateMutex(+ host: StackStateHost,+ project: string+): Promise<{ release: () => Promise<void> }> {+ await writeStateVolume(host, 'true', '', []);+ const holding = run('docker', [+ 'run',+ '--rm',+ '--volume',+ `${project}_${STACK_STATE_VOLUME_SUFFIX}:/state`,+ '--entrypoint',+ 'sh',+ STACK_HELPER_IMAGE,+ '-c',+ 'exec 9>/state/.mutex; flock 9; touch /state/held; ' ++ 'while [ ! -f /state/release ]; do sleep 0.1; done; rm /state/held /state/release',+ ]);+ while ((await readStateVolume(host, '[ -f /state/held ] && echo held; true')) !== 'held\n') {+ await sleep(100);+ }+ return {+ release: async () => {+ await runStateScript(host, 'touch /state/release', []);+ await holding;+ },+ };+}+describe.skipIf(!DOCKER)('the server lock, against a real state volume', () => {let host: StackStateHost;let project: string;@@ -116,7 +164,7 @@ describe.skipIf(!DOCKER)('the server lock, against a real state volume', () => {);});- async function take(c: LockClaim, timing: LockTiming = QUIET): Promise<ServerLease> {+ async function take(c: LockClaim, timing: LockTiming = HELD): Promise<ServerLease> {const lease = await acquireServerLock(host, c, { mode: 'refuse', timing });leases.push(lease);return lease;@@ -153,7 +201,7 @@ describe.skipIf(!DOCKER)('the server lock, against a real state volume', () => {const onWaiting = vi.fn();const waiting = acquireServerLock(host, bob(), {mode: 'wait',- timing: QUIET,+ timing: HELD,signal: new AbortController().signal,onWaiting,});@@ -166,7 +214,7 @@ describe.skipIf(!DOCKER)('the server lock, against a real state volume', () => {}, 60_000);it('can be taken over once its holder stops renewing it', async () => {- await take(claim());+ await take(claim(), LAPSING);await sleep(2500);@@ -174,7 +222,7 @@ describe.skipIf(!DOCKER)('the server lock, against a real state volume', () => {}, 60_000);it('goes to exactly one of two Consoles taking over the same lapsed lease', async () => {- await take(claim());+ await take(claim(), LAPSING);await sleep(2500);const results = await Promise.allSettled([@@ -186,15 +234,17 @@ describe.skipIf(!DOCKER)('the server lock, against a real state volume', () => {}, 60_000);it('stays held past its lease while its holder renews it', async () => {- await take(claim(), { ...QUIET, renewEveryMs: 500 });+ // Long enough a lease that a renewal always lands inside it, even when+ // starting each renewal's container is slow.+ await take(claim(), { ...HELD, ttlSeconds: 5, renewEveryMs: 500 });- await sleep(3000);+ await sleep(6500);await expect(take(bob())).rejects.toBeInstanceOf(ServerBusyError);}, 60_000);it('cannot be renewed past the longest hold, and then goes to whoever asks', async () => {- const alice = await take(claim(), { ...QUIET, renewEveryMs: 500, maxHoldSeconds: 3 });+ const alice = await take(claim(), { ...LAPSING, renewEveryMs: 500, maxHoldSeconds: 3 });await sleep(4000);@@ -203,7 +253,7 @@ describe.skipIf(!DOCKER)('the server lock, against a real state volume', () => {}, 60_000);it('is not released by a holder that lost it, and knows it was lost', async () => {- const alice = await take(claim());+ const alice = await take(claim(), LAPSING);await sleep(2500);await take(bob());@@ -257,27 +307,16 @@ describe.skipIf(!DOCKER)('the server lock, against a real state volume', () => {}, 60_000);it('is only taken under the state mutex', async () => {- await writeStateVolume(host, 'true', '', []);- const holding = run('docker', [- 'run',- '--rm',- '--volume',- `${project}_${STACK_STATE_VOLUME_SUFFIX}:/state`,- '--entrypoint',- 'sh',- STACK_HELPER_IMAGE,- '-c',- 'exec 9>/state/.mutex; flock 9; touch /state/held; sleep 3; rm /state/held',- ]);- while ((await readStateVolume(host, '[ -f /state/held ] && echo held; true')) !== 'held\n') {- await sleep(100);+ const holder = await holdStateMutex(host, project);++ const taking = settles(take(claim()));+ try {+ await sleep(1500);+ expect(taking.settled()).toBe(false);+ } finally {+ await holder.release();}-- const started = Date.now();- await take(claim());-- expect(Date.now() - started).toBeGreaterThanOrEqual(1500);- await holding;+ await taking.promise;}, 60_000);it('is not taken when its file cannot be written, so nobody else takes it too', async () => {@@ -300,7 +339,7 @@ describe.skipIf(!DOCKER)('the server lock, against a real state volume', () => {}, 60_000);it('lets only its holder publish or withdraw the settings', async () => {- const alice = await take(claim());+ const alice = await take(claim(), LAPSING);await sleep(2500);const bobs = await take(bob());@@ -329,31 +368,22 @@ describe.skipIf(!DOCKER)('the state mutex, against a real state volume', () => {it('makes a record wait while another script holds the mutex', async () => {// Container start-up jitter dwarfs a trim, so racing real records rarely- // shows anything; holding the mutex for a known time does.- await writeStateVolume(host, 'true', '', []);- const holding = run('docker', [- 'run',- '--rm',- '--volume',- `${project}_${STACK_STATE_VOLUME_SUFFIX}:/state`,- '--entrypoint',- 'sh',- STACK_HELPER_IMAGE,- '-c',- 'exec 9>/state/.mutex; flock 9; touch /state/held; sleep 3; rm /state/held',- ]);- while ((await readStateVolume(host, '[ -f /state/held ] && echo held; true')) !== 'held\n') {- await sleep(100);+ // shows anything; holding the mutex until told to let go does.+ const holder = await holdStateMutex(host, project);++ const recording = settles(+ writeStateVolume(host, RECORD_SCRIPT, '{"entry":1}\n{"line":1}\n', [+ 'dddddddd-0000-4000-8000-000000000001',+ 'act',+ ])+ );+ try {+ await sleep(1500);+ expect(recording.settled()).toBe(false);+ } finally {+ await holder.release();}-- const started = Date.now();- await writeStateVolume(host, RECORD_SCRIPT, '{"entry":1}\n{"line":1}\n', [- 'dddddddd-0000-4000-8000-000000000001',- 'act',- ]);-- expect(Date.now() - started).toBeGreaterThanOrEqual(1500);+ await recording.promise;expect(await readStateVolume(host, '[ -f /state/held ] && echo held; true')).toBe('');- await holding;}, 60_000);});diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/stack-state.test.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/stack-state.test.tsindex c2e76bb91..f76d4b392 100644--- a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/stack-state.test.ts+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/stack-state.test.ts@@ -1,5 +1,5 @@import { execFileSync } from 'node:child_process';-import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';+import { closeSync, mkdtempSync, openSync, readFileSync, rmSync, writeFileSync } from 'node:fs';import { tmpdir } from 'node:os';import { dirname, join } from 'node:path';import { fileURLToPath } from 'node:url';@@ -410,13 +410,16 @@ describe('the published copy', () => {describe('the published copy’s scripts, run for real', () => {let dir: string;+ let stdinDir: string;beforeEach(() => {dir = mkdtempSync(join(tmpdir(), 'stack-state-'));+ stdinDir = mkdtempSync(join(tmpdir(), 'stack-state-stdin-'));});afterEach(() => {rmSync(dir, { recursive: true, force: true });+ rmSync(stdinDir, { recursive: true, force: true });});/** A host whose state volume is `dir`, with every script run by `sh`. */@@ -428,11 +431,21 @@ describe('the published copy’s scripts, run for real', () => {const run = dockerRun(args);const at = run.indexOf('-c');const script = run[at + 1]!.replaceAll('/state', dir);- return execFileSync('sh', ['-c', script, ...run.slice(at + 2)], {- input,- encoding: 'utf8',- env,- });+ // Stdin is a file, not a pipe: a script that refuses before reading its+ // input (the lock check) can exit before a piped write lands, and the+ // write then fails with EPIPE in place of the script's own error.+ const inputPath = join(stdinDir, 'input');+ writeFileSync(inputPath, input);+ const stdin = openSync(inputPath, 'r');+ try {+ return execFileSync('sh', ['-c', script, ...run.slice(at + 2)], {+ stdio: [stdin, 'pipe', 'pipe'],+ encoding: 'utf8',+ env,+ });+ } finally {+ closeSync(stdin);+ }};fake.writeCommandInput.mockImplementation(async (_command, args, input) => {sh(args, input);diff --git a/console/apps/switch-console-desktop/src/main/core/ssh/connect/resolve-ssh-connect-config.test.ts b/console/apps/switch-console-desktop/src/main/core/ssh/connect/resolve-ssh-connect-config.test.tsindex ad83e80e2..188df6968 100644--- a/console/apps/switch-console-desktop/src/main/core/ssh/connect/resolve-ssh-connect-config.test.ts+++ b/console/apps/switch-console-desktop/src/main/core/ssh/connect/resolve-ssh-connect-config.test.ts@@ -81,6 +81,15 @@ function parseOtherFixturePublicKey(): ParsedKey {return parsed;}+/**+ * ECDSA rather than ed25519: ssh2's ed25519 generator strips a leading zero+ * byte from the public key, so about one ed25519 key in 256 comes out+ * malformed and `parseKey` rejects it. Its ECDSA keys always parse.+ */+function generateParseableKeyPair() {+ return utils.generateKeyPairSync('ecdsa', { bits: 256 });+}+describe('resolveSshConnectConfig', () => {it('uses ssh -G as authoritative for alias-backed ProxyCommand', async () => {const spawned: string[] = [];@@ -682,7 +691,7 @@ describe('resolveSshConnectConfig', () => {});it('offers an unencrypted IdentityFile key from disk alongside the agent (gcloud/IAP)', async () => {- const keyPair = utils.generateKeyPairSync('ed25519');+ const keyPair = generateParseableKeyPair();const readFiles: string[] = [];const result = await resolveSshConnectConfig({@@ -718,7 +727,7 @@ describe('resolveSshConnectConfig', () => {});it('uses an on-disk IdentityFile key when the agent socket is unavailable', async () => {- const keyPair = utils.generateKeyPairSync('ed25519');+ const keyPair = generateParseableKeyPair();const result = await resolveSshConnectConfig({kind: 'transient',diff --git a/console/packages/agent-controller/src/runtime.ts b/console/packages/agent-controller/src/runtime.tsindex cd7536e69..8052ebaa6 100644--- a/console/packages/agent-controller/src/runtime.ts+++ b/console/packages/agent-controller/src/runtime.ts@@ -550,6 +550,9 @@ export class InProcessRuntime implements AgentRuntime {const sessions = inProcessSupervision(this.deps.bundlePath, this.links);// For `status`, which runs in another process and reads only disk.const stopRecording = recordWatcherHealth(root, control, null);+ const forget = () => {+ if (this.hosts.get(agentId)?.done === done) this.hosts.delete(agentId);+ };const done = (async () => {try {await runAgentHost(root, config, signal, sessions, control, this.deps.openStream(agentId));@@ -563,22 +566,27 @@ export class InProcessRuntime implements AgentRuntime {this.deps.log.info('Agent host stopped', { agentId });return false;},- (error: unknown) => this.crashed(agentId, root, signal, error)+ (error: unknown) => this.crashed(agentId, root, signal, error, forget)).then((again) => {- if (this.hosts.get(agentId)?.done === done) this.hosts.delete(agentId);+ forget();if (again) this.start(agentId, root, config);});this.hosts.set(agentId, { stop, done, control, sessions });this.deps.log.info('Agent host started', { agentId });}- /** Whether an agent host that failed is started again. */+ /**+ * Whether an agent host that failed is started again. One that is not is+ * forgotten before its failure is recorded, so nothing that reads the+ * failure can still see it alive.+ */private async crashed(agentId: string,root: string,signal: AbortSignal,- error: unknown+ error: unknown,+ forget: () => void): Promise<boolean> {const message = errorMessage(error);if (signal.aborted) {@@ -594,6 +602,7 @@ export class InProcessRuntime implements AgentRuntime {agentId,error: message,});+ forget();await mkdir(join(root, 'supervisor'), { recursive: true, mode: 0o700 });await writeAtomic(join(root, 'supervisor', 'failure.json'), JSON.stringify({ message }));return false;