Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

fix(switchdash): stop the readiness probe racing itself (CHOO-1809)

Louis Amaudruz4272a30791ca
diff --git a/dash/apps/switchdash-desktop/src/renderer/features/locations/components/add-agent-modal/add-agent-modal.tsx b/dash/apps/switchdash-desktop/src/renderer/features/locations/components/add-agent-modal/add-agent-modal.tsx
index fc2fda7ab..b9d377714 100644
--- a/dash/apps/switchdash-desktop/src/renderer/features/locations/components/add-agent-modal/add-agent-modal.tsx
+++ b/dash/apps/switchdash-desktop/src/renderer/features/locations/components/add-agent-modal/add-agent-modal.tsx
@@ -678,7 +678,10 @@ export const AddAgentModal = observer(function AddAgentModal({ onClose }: AddLoc
onNavigateAway={onClose}
/>
)}
- {isRemoteRun && canChooseAgentType && (
+ {/* Not while we are still finding out what the host has: asking for a
+ working directory under a "checking…" spinner invites the user to
+ fill in a form we may be about to refuse. */}
+ {isRemoteRun && canChooseAgentType && !hostReadiness.checking && (
<div className="flex items-center gap-2">
<Input
value={remoteRepoDirDraft}
diff --git a/dash/apps/switchdash-desktop/src/renderer/features/remote-hosts/host-probe-queue.test.ts b/dash/apps/switchdash-desktop/src/renderer/features/remote-hosts/host-probe-queue.test.ts
new file mode 100644
index 000000000..5a9657735
--- /dev/null
+++ b/dash/apps/switchdash-desktop/src/renderer/features/remote-hosts/host-probe-queue.test.ts
@@ -0,0 +1,99 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { queueHostProbe, resetHostProbeQueue } from './host-probe-queue';
+
+/**
+ * The setup runner takes one operation per host and refuses the rest (CHOO-1809).
+ *
+ * Reported as a flood of `Another setup operation is already running for
+ * bench-vm` for steps the user never touched. The cause was overlap, not the
+ * host: the readiness gate probed the steps it found stale, and each completed
+ * step pushed a new plan, which shortened the stale list, which re-rendered the
+ * caller, which asked again while the first pass was still going.
+ */
+beforeEach(() => resetHostProbeQueue());
+
+/** A task that reports whether it ever ran while another was in flight. */
+function overlapDetector() {
+ const state = { running: 0, maxConcurrent: 0, order: [] as string[] };
+ const task =
+ (label: string, ms = 0) =>
+ async () => {
+ state.running += 1;
+ state.maxConcurrent = Math.max(state.maxConcurrent, state.running);
+ state.order.push(label);
+ await new Promise((resolve) => setTimeout(resolve, ms));
+ state.running -= 1;
+ return label;
+ };
+ return { state, task };
+}
+
+describe('queueHostProbe', () => {
+ it('never runs two probes for a host at once', async () => {
+ const { state, task } = overlapDetector();
+
+ await Promise.all([
+ queueHostProbe('bench-vm', task('a', 5)),
+ queueHostProbe('bench-vm', task('b', 5)),
+ queueHostProbe('bench-vm', task('c', 5)),
+ ]);
+
+ expect(state.maxConcurrent).toBe(1);
+ });
+
+ it('runs them in the order they were asked for', async () => {
+ const { state, task } = overlapDetector();
+
+ await Promise.all([
+ queueHostProbe('bench-vm', task('a', 5)),
+ queueHostProbe('bench-vm', task('b', 1)),
+ ]);
+
+ expect(state.order).toEqual(['a', 'b']);
+ });
+
+ it('does not make one host wait for another', async () => {
+ // Hosts are independent; the runner's lock is per host. Serialising across
+ // hosts would make a slow VM hold up every other machine.
+ const { state, task } = overlapDetector();
+
+ await Promise.all([
+ queueHostProbe('bench-vm', task('a', 5)),
+ queueHostProbe('dev-vm', task('b', 5)),
+ ]);
+
+ expect(state.maxConcurrent).toBe(2);
+ });
+
+ it('runs the next probe even after one fails', async () => {
+ // A failed probe is ordinary — an unreachable host, a refused command. It
+ // must not strand everything queued behind it.
+ const after = vi.fn(async () => 'ran');
+
+ const failing = queueHostProbe('bench-vm', async () => {
+ throw new Error('unreachable');
+ });
+ const next = queueHostProbe('bench-vm', after);
+
+ await expect(failing).rejects.toThrow('unreachable');
+ await expect(next).resolves.toBe('ran');
+ expect(after).toHaveBeenCalledOnce();
+ });
+
+ it('gives the caller its own error, not the previous task’s', async () => {
+ const failing = queueHostProbe('bench-vm', async () => {
+ throw new Error('first');
+ });
+ await expect(failing).rejects.toThrow('first');
+
+ await expect(
+ queueHostProbe('bench-vm', async () => {
+ throw new Error('second');
+ })
+ ).rejects.toThrow('second');
+ });
+
+ it('returns the task’s own value', async () => {
+ await expect(queueHostProbe('bench-vm', async () => 42)).resolves.toBe(42);
+ });
+});
diff --git a/dash/apps/switchdash-desktop/src/renderer/features/remote-hosts/host-probe-queue.ts b/dash/apps/switchdash-desktop/src/renderer/features/remote-hosts/host-probe-queue.ts
new file mode 100644
index 000000000..7307ab221
--- /dev/null
+++ b/dash/apps/switchdash-desktop/src/renderer/features/remote-hosts/host-probe-queue.ts
@@ -0,0 +1,46 @@
+/**
+ * One host probe at a time, across every caller (CHOO-1809).
+ *
+ * The setup runner accepts one operation per host and refuses the rest outright,
+ * so two callers asking at once do not interleave — the loser throws "another
+ * setup operation is already running". That reads as a fault on the host when it
+ * is really a queueing failure on our side, and it filled the log with stacks
+ * for steps nobody had asked about.
+ *
+ * Overlap is easy to cause without meaning to. The readiness gate probes the
+ * steps it found stale, and every completed step pushes a new plan to the
+ * renderer — which shortens the stale list, which re-renders the caller, which
+ * asks again while the first pass is still running. Serialising here fixes that
+ * class of collision wherever it comes from, rather than in each caller.
+ */
+
+const queues = new Map<string, Promise<unknown>>();
+
+/**
+ * Run `task` once every probe already queued for this host has settled.
+ *
+ * Failures do not poison the queue: the next task runs regardless of how the
+ * previous one ended, while the caller still sees its own error.
+ */
+export function queueHostProbe<T>(sshHost: string, task: () => Promise<T>): Promise<T> {
+ const previous = queues.get(sshHost) ?? Promise.resolve();
+ // Same continuation for both outcomes — a rejected predecessor must not skip
+ // the work behind it.
+ const next = previous.then(task, task);
+ queues.set(
+ sshHost,
+ next.catch(() => undefined)
+ );
+ return next;
+}
+
+/** Whether anything is queued for this host. Exposed for tests. */
+export function hasQueuedProbe(sshHost: string): boolean {
+ return queues.has(sshHost);
+}
+
+/** Drop a host's queue. Tests only — the queue is otherwise self-clearing. */
+export function resetHostProbeQueue(sshHost?: string): void {
+ if (sshHost) queues.delete(sshHost);
+ else queues.clear();
+}
diff --git a/dash/apps/switchdash-desktop/src/renderer/features/remote-hosts/host-readiness-notice.tsx b/dash/apps/switchdash-desktop/src/renderer/features/remote-hosts/host-readiness-notice.tsx
index 495399f75..da6f07a21 100644
--- a/dash/apps/switchdash-desktop/src/renderer/features/remote-hosts/host-readiness-notice.tsx
+++ b/dash/apps/switchdash-desktop/src/renderer/features/remote-hosts/host-readiness-notice.tsx
@@ -21,6 +21,7 @@ import { Button } from '@renderer/lib/ui/button';
import { Spinner } from '@renderer/lib/ui/spinner';
import { log } from '@renderer/utils/logger';
import { deriveAgentTypeStatus } from '@shared/core/remote-hosts/host-status';
+import { queueHostProbe } from './host-probe-queue';
import { hostReachabilityStore } from './host-reachability-store';
import { resolveReadiness, stepsNeedingObservation, type HostReadiness } from './host-readiness';
import { hostSetupStore } from './host-setup-store';
@@ -40,8 +41,12 @@ export function useRemoteHostReadiness(
agentId: string | null
): HostReadiness {
const [checking, setChecking] = useState(false);
- // One probe per host+type per mount. Two modals open on the same host would
- // otherwise queue behind each other on a runner that takes one job at a time.
+ // One probe per host+type per mount. Keyed on the host and type ALONE: an
+ // earlier version folded the stale-step list into this key, and since every
+ // completed step pushes a new plan — shortening that list — the key changed
+ // mid-pass and started a second run over the first. The runner refused it, and
+ // the log filled with "another setup operation is already running" for steps
+ // nobody had asked about.
const probed = useRef(new Set<string>());
useEffect(() => {
@@ -56,26 +61,33 @@ export function useRemoteHostReadiness(
// The persisted plan answers this most of the time. Only what is missing or
// gone stale is re-observed, and only for this agent type.
const stale = sshHost ? stepsNeedingObservation(plan, agentId, Date.now()) : [];
+ // Held in a ref so the effect can read the current list without depending on
+ // it — the list is derived from a plan that changes while the probe runs.
+ const staleRef = useRef(stale);
+ staleRef.current = stale;
+
// No plan at all is a different job: there is nothing to refresh, the host
// has to be surveyed from scratch.
const needsSurvey = !!sshHost && !plan;
const needsProbe = needsSurvey || stale.length > 0;
- const probeKey = `${sshHost}:${agentId ?? '*'}:${stale.join(',')}`;
+ const probeKey = `${sshHost}:${agentId ?? '*'}`;
useEffect(() => {
if (!sshHost || !needsProbe || probed.current.has(probeKey)) return;
probed.current.add(probeKey);
setChecking(true);
- const run = needsSurvey
- ? rpc.remoteHosts.recheckSetup(sshHost)
- : // Sequentially: the runner takes one operation per host at a time, so
- // firing these together only makes the later ones fail.
- stale.reduce<Promise<unknown>>(
- (queue, stepId) =>
- queue.then(() => rpc.remoteHosts.recheckSetupStep({ sshHost, stepId })),
- Promise.resolve()
- );
- run
+ const steps = staleRef.current;
+ queueHostProbe(sshHost, async () => {
+ if (needsSurvey) {
+ await rpc.remoteHosts.recheckSetup(sshHost);
+ return;
+ }
+ // Sequentially, and behind the queue: the runner takes one operation per
+ // host, so overlapping requests only make the later ones fail.
+ for (const stepId of steps) {
+ await rpc.remoteHosts.recheckSetupStep({ sshHost, stepId });
+ }
+ })
.catch((error: unknown) => {
// A probe we could not run tells us nothing. Leaving the gate open is
// the honest outcome — the alternative blocks the user over our own
@@ -83,10 +95,7 @@ export function useRemoteHostReadiness(
log.warn('Could not check host readiness before creating an agent', { sshHost, error });
})
.finally(() => setChecking(false));
- // `stale` is derived from the plan and changes identity every render; the
- // key already encodes its contents.
- // eslint-disable-next-line react-hooks/exhaustive-deps
- }, [sshHost, needsProbe, needsSurvey, probeKey]);
+ }, [sshHost, probeKey, needsProbe, needsSurvey]);
return resolveReadiness(status, plan, agentId, checking || needsProbe);
}