Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

Make the Console tests that failed intermittently in CI deterministic

Petr Bauch273735f5e638
diff --git a/console/apps/switch-console-desktop/src/main/core/agents/remote-dir.integration.test.ts b/console/apps/switch-console-desktop/src/main/core/agents/remote-dir.integration.test.ts
index f20bf64f2..12a4455da 100644
--- a/console/apps/switch-console-desktop/src/main/core/agents/remote-dir.integration.test.ts
+++ b/console/apps/switch-console-desktop/src/main/core/agents/remote-dir.integration.test.ts
@@ -44,7 +44,9 @@ vi.mock('@main/core/locations/location-transport', () => ({
const { inspectRemoteDir } = await import('./remote-dir');
beforeAll(async () => {
- const { private: hostKey } = utils.generateKeyPairSync('ed25519');
+ // Not ed25519: ssh2's ed25519 generator strips a leading zero byte from the
+ // public key, so about one key in 256 is malformed. ECDSA keys always parse.
+ const { private: hostKey } = utils.generateKeyPairSync('ecdsa', { bits: 256 });
server = new Server({ hostKeys: [hostKey] }, (conn) => {
conn.on('authentication', (ctx) => ctx.accept());
diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/stack-lock.docker.test.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/stack-lock.docker.test.ts
index 12d5132a1..0a899af54 100644
--- a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/stack-lock.docker.test.ts
+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/stack-lock.docker.test.ts
@@ -89,16 +89,64 @@ const bob = (overrides: Partial<LockClaim> = {}) =>
...overrides,
});
-/** No renewals unless a test asks for them, so expiry can be watched. */
-const QUIET: LockTiming = {
- ttlSeconds: 2,
+/**
+ * A lease that outlasts any test, for tests that need the lock to stay held
+ * while they act on it. Each step here starts a container, so a short lease
+ * can lapse between two of them on a slow machine.
+ */
+const HELD: LockTiming = {
+ ttlSeconds: 600,
renewEveryMs: 60_000,
maxHoldSeconds: 600,
pollEveryMs: 200,
};
+/** No renewals unless a test asks for them, so expiry can be watched. */
+const LAPSING: LockTiming = { ...HELD, ttlSeconds: 2 };
+
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+/** `promise`, and whether it has settled yet. */
+function settles<T>(promise: Promise<T>): { promise: Promise<T>; settled: () => boolean } {
+ let settled = false;
+ const tracked = promise.finally(() => {
+ settled = true;
+ });
+ return { promise: tracked, settled: () => settled };
+}
+
+/**
+ * Holds the state mutex from a container of its own until released, so a test
+ * can show a script waiting for it however long containers take to start.
+ */
+async function holdStateMutex(
+ host: StackStateHost,
+ project: string
+): Promise<{ release: () => Promise<void> }> {
+ await writeStateVolume(host, 'true', '', []);
+ const holding = run('docker', [
+ 'run',
+ '--rm',
+ '--volume',
+ `${project}_${STACK_STATE_VOLUME_SUFFIX}:/state`,
+ '--entrypoint',
+ 'sh',
+ STACK_HELPER_IMAGE,
+ '-c',
+ 'exec 9>/state/.mutex; flock 9; touch /state/held; ' +
+ 'while [ ! -f /state/release ]; do sleep 0.1; done; rm /state/held /state/release',
+ ]);
+ while ((await readStateVolume(host, '[ -f /state/held ] && echo held; true')) !== 'held\n') {
+ await sleep(100);
+ }
+ return {
+ release: async () => {
+ await runStateScript(host, 'touch /state/release', []);
+ await holding;
+ },
+ };
+}
+
describe.skipIf(!DOCKER)('the server lock, against a real state volume', () => {
let host: StackStateHost;
let project: string;
@@ -116,7 +164,7 @@ describe.skipIf(!DOCKER)('the server lock, against a real state volume', () => {
);
});
- async function take(c: LockClaim, timing: LockTiming = QUIET): Promise<ServerLease> {
+ async function take(c: LockClaim, timing: LockTiming = HELD): Promise<ServerLease> {
const lease = await acquireServerLock(host, c, { mode: 'refuse', timing });
leases.push(lease);
return lease;
@@ -153,7 +201,7 @@ describe.skipIf(!DOCKER)('the server lock, against a real state volume', () => {
const onWaiting = vi.fn();
const waiting = acquireServerLock(host, bob(), {
mode: 'wait',
- timing: QUIET,
+ timing: HELD,
signal: new AbortController().signal,
onWaiting,
});
@@ -166,7 +214,7 @@ describe.skipIf(!DOCKER)('the server lock, against a real state volume', () => {
}, 60_000);
it('can be taken over once its holder stops renewing it', async () => {
- await take(claim());
+ await take(claim(), LAPSING);
await sleep(2500);
@@ -174,7 +222,7 @@ describe.skipIf(!DOCKER)('the server lock, against a real state volume', () => {
}, 60_000);
it('goes to exactly one of two Consoles taking over the same lapsed lease', async () => {
- await take(claim());
+ await take(claim(), LAPSING);
await sleep(2500);
const results = await Promise.allSettled([
@@ -186,15 +234,17 @@ describe.skipIf(!DOCKER)('the server lock, against a real state volume', () => {
}, 60_000);
it('stays held past its lease while its holder renews it', async () => {
- await take(claim(), { ...QUIET, renewEveryMs: 500 });
+ // Long enough a lease that a renewal always lands inside it, even when
+ // starting each renewal's container is slow.
+ await take(claim(), { ...HELD, ttlSeconds: 5, renewEveryMs: 500 });
- await sleep(3000);
+ await sleep(6500);
await expect(take(bob())).rejects.toBeInstanceOf(ServerBusyError);
}, 60_000);
it('cannot be renewed past the longest hold, and then goes to whoever asks', async () => {
- const alice = await take(claim(), { ...QUIET, renewEveryMs: 500, maxHoldSeconds: 3 });
+ const alice = await take(claim(), { ...LAPSING, renewEveryMs: 500, maxHoldSeconds: 3 });
await sleep(4000);
@@ -203,7 +253,7 @@ describe.skipIf(!DOCKER)('the server lock, against a real state volume', () => {
}, 60_000);
it('is not released by a holder that lost it, and knows it was lost', async () => {
- const alice = await take(claim());
+ const alice = await take(claim(), LAPSING);
await sleep(2500);
await take(bob());
@@ -257,27 +307,16 @@ describe.skipIf(!DOCKER)('the server lock, against a real state volume', () => {
}, 60_000);
it('is only taken under the state mutex', async () => {
- await writeStateVolume(host, 'true', '', []);
- const holding = run('docker', [
- 'run',
- '--rm',
- '--volume',
- `${project}_${STACK_STATE_VOLUME_SUFFIX}:/state`,
- '--entrypoint',
- 'sh',
- STACK_HELPER_IMAGE,
- '-c',
- 'exec 9>/state/.mutex; flock 9; touch /state/held; sleep 3; rm /state/held',
- ]);
- while ((await readStateVolume(host, '[ -f /state/held ] && echo held; true')) !== 'held\n') {
- await sleep(100);
+ const holder = await holdStateMutex(host, project);
+
+ const taking = settles(take(claim()));
+ try {
+ await sleep(1500);
+ expect(taking.settled()).toBe(false);
+ } finally {
+ await holder.release();
}
-
- const started = Date.now();
- await take(claim());
-
- expect(Date.now() - started).toBeGreaterThanOrEqual(1500);
- await holding;
+ await taking.promise;
}, 60_000);
it('is not taken when its file cannot be written, so nobody else takes it too', async () => {
@@ -300,7 +339,7 @@ describe.skipIf(!DOCKER)('the server lock, against a real state volume', () => {
}, 60_000);
it('lets only its holder publish or withdraw the settings', async () => {
- const alice = await take(claim());
+ const alice = await take(claim(), LAPSING);
await sleep(2500);
const bobs = await take(bob());
@@ -329,31 +368,22 @@ describe.skipIf(!DOCKER)('the state mutex, against a real state volume', () => {
it('makes a record wait while another script holds the mutex', async () => {
// Container start-up jitter dwarfs a trim, so racing real records rarely
- // shows anything; holding the mutex for a known time does.
- await writeStateVolume(host, 'true', '', []);
- const holding = run('docker', [
- 'run',
- '--rm',
- '--volume',
- `${project}_${STACK_STATE_VOLUME_SUFFIX}:/state`,
- '--entrypoint',
- 'sh',
- STACK_HELPER_IMAGE,
- '-c',
- 'exec 9>/state/.mutex; flock 9; touch /state/held; sleep 3; rm /state/held',
- ]);
- while ((await readStateVolume(host, '[ -f /state/held ] && echo held; true')) !== 'held\n') {
- await sleep(100);
+ // shows anything; holding the mutex until told to let go does.
+ const holder = await holdStateMutex(host, project);
+
+ const recording = settles(
+ writeStateVolume(host, RECORD_SCRIPT, '{"entry":1}\n{"line":1}\n', [
+ 'dddddddd-0000-4000-8000-000000000001',
+ 'act',
+ ])
+ );
+ try {
+ await sleep(1500);
+ expect(recording.settled()).toBe(false);
+ } finally {
+ await holder.release();
}
-
- const started = Date.now();
- await writeStateVolume(host, RECORD_SCRIPT, '{"entry":1}\n{"line":1}\n', [
- 'dddddddd-0000-4000-8000-000000000001',
- 'act',
- ]);
-
- expect(Date.now() - started).toBeGreaterThanOrEqual(1500);
+ await recording.promise;
expect(await readStateVolume(host, '[ -f /state/held ] && echo held; true')).toBe('');
- await holding;
}, 60_000);
});
diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/stack-state.test.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/stack-state.test.ts
index c2e76bb91..f76d4b392 100644
--- a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/stack-state.test.ts
+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/stack-state.test.ts
@@ -1,5 +1,5 @@
import { execFileSync } from 'node:child_process';
-import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { closeSync, mkdtempSync, openSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
@@ -410,13 +410,16 @@ describe('the published copy', () => {
describe('the published copy’s scripts, run for real', () => {
let dir: string;
+ let stdinDir: string;
beforeEach(() => {
dir = mkdtempSync(join(tmpdir(), 'stack-state-'));
+ stdinDir = mkdtempSync(join(tmpdir(), 'stack-state-stdin-'));
});
afterEach(() => {
rmSync(dir, { recursive: true, force: true });
+ rmSync(stdinDir, { recursive: true, force: true });
});
/** A host whose state volume is `dir`, with every script run by `sh`. */
@@ -428,11 +431,21 @@ describe('the published copy’s scripts, run for real', () => {
const run = dockerRun(args);
const at = run.indexOf('-c');
const script = run[at + 1]!.replaceAll('/state', dir);
- return execFileSync('sh', ['-c', script, ...run.slice(at + 2)], {
- input,
- encoding: 'utf8',
- env,
- });
+ // Stdin is a file, not a pipe: a script that refuses before reading its
+ // input (the lock check) can exit before a piped write lands, and the
+ // write then fails with EPIPE in place of the script's own error.
+ const inputPath = join(stdinDir, 'input');
+ writeFileSync(inputPath, input);
+ const stdin = openSync(inputPath, 'r');
+ try {
+ return execFileSync('sh', ['-c', script, ...run.slice(at + 2)], {
+ stdio: [stdin, 'pipe', 'pipe'],
+ encoding: 'utf8',
+ env,
+ });
+ } finally {
+ closeSync(stdin);
+ }
};
fake.writeCommandInput.mockImplementation(async (_command, args, input) => {
sh(args, input);
diff --git a/console/apps/switch-console-desktop/src/main/core/ssh/connect/resolve-ssh-connect-config.test.ts b/console/apps/switch-console-desktop/src/main/core/ssh/connect/resolve-ssh-connect-config.test.ts
index ad83e80e2..188df6968 100644
--- a/console/apps/switch-console-desktop/src/main/core/ssh/connect/resolve-ssh-connect-config.test.ts
+++ b/console/apps/switch-console-desktop/src/main/core/ssh/connect/resolve-ssh-connect-config.test.ts
@@ -81,6 +81,15 @@ function parseOtherFixturePublicKey(): ParsedKey {
return parsed;
}
+/**
+ * ECDSA rather than ed25519: ssh2's ed25519 generator strips a leading zero
+ * byte from the public key, so about one ed25519 key in 256 comes out
+ * malformed and `parseKey` rejects it. Its ECDSA keys always parse.
+ */
+function generateParseableKeyPair() {
+ return utils.generateKeyPairSync('ecdsa', { bits: 256 });
+}
+
describe('resolveSshConnectConfig', () => {
it('uses ssh -G as authoritative for alias-backed ProxyCommand', async () => {
const spawned: string[] = [];
@@ -682,7 +691,7 @@ describe('resolveSshConnectConfig', () => {
});
it('offers an unencrypted IdentityFile key from disk alongside the agent (gcloud/IAP)', async () => {
- const keyPair = utils.generateKeyPairSync('ed25519');
+ const keyPair = generateParseableKeyPair();
const readFiles: string[] = [];
const result = await resolveSshConnectConfig(
{
@@ -718,7 +727,7 @@ describe('resolveSshConnectConfig', () => {
});
it('uses an on-disk IdentityFile key when the agent socket is unavailable', async () => {
- const keyPair = utils.generateKeyPairSync('ed25519');
+ const keyPair = generateParseableKeyPair();
const result = await resolveSshConnectConfig(
{
kind: 'transient',
diff --git a/console/packages/agent-controller/src/runtime.ts b/console/packages/agent-controller/src/runtime.ts
index cd7536e69..8052ebaa6 100644
--- a/console/packages/agent-controller/src/runtime.ts
+++ b/console/packages/agent-controller/src/runtime.ts
@@ -550,6 +550,9 @@ export class InProcessRuntime implements AgentRuntime {
const sessions = inProcessSupervision(this.deps.bundlePath, this.links);
// For `status`, which runs in another process and reads only disk.
const stopRecording = recordWatcherHealth(root, control, null);
+ const forget = () => {
+ if (this.hosts.get(agentId)?.done === done) this.hosts.delete(agentId);
+ };
const done = (async () => {
try {
await runAgentHost(root, config, signal, sessions, control, this.deps.openStream(agentId));
@@ -563,22 +566,27 @@ export class InProcessRuntime implements AgentRuntime {
this.deps.log.info('Agent host stopped', { agentId });
return false;
},
- (error: unknown) => this.crashed(agentId, root, signal, error)
+ (error: unknown) => this.crashed(agentId, root, signal, error, forget)
)
.then((again) => {
- if (this.hosts.get(agentId)?.done === done) this.hosts.delete(agentId);
+ forget();
if (again) this.start(agentId, root, config);
});
this.hosts.set(agentId, { stop, done, control, sessions });
this.deps.log.info('Agent host started', { agentId });
}
- /** Whether an agent host that failed is started again. */
+ /**
+ * Whether an agent host that failed is started again. One that is not is
+ * forgotten before its failure is recorded, so nothing that reads the
+ * failure can still see it alive.
+ */
private async crashed(
agentId: string,
root: string,
signal: AbortSignal,
- error: unknown
+ error: unknown,
+ forget: () => void
): Promise<boolean> {
const message = errorMessage(error);
if (signal.aborted) {
@@ -594,6 +602,7 @@ export class InProcessRuntime implements AgentRuntime {
agentId,
error: message,
});
+ forget();
await mkdir(join(root, 'supervisor'), { recursive: true, mode: 0o700 });
await writeAtomic(join(root, 'supervisor', 'failure.json'), JSON.stringify({ message }));
return false;