The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
Test adapters in the environment a run would actually use (#10698)
Devin Foley2ffebd48363b
diff --git a/ui/src/components/AgentConfigForm.tsx b/ui/src/components/AgentConfigForm.tsxindex bbaa9cf67f..18bbaff7df 100644--- a/ui/src/components/AgentConfigForm.tsx+++ b/ui/src/components/AgentConfigForm.tsx@@ -27,6 +27,7 @@ import {import { Button } from "@/components/ui/button";import { FolderOpen, Heart, ChevronDown, X } from "lucide-react";import { asBoolean, asFiniteNumber, asObject, cn } from "../lib/utils";+import { resolveAdapterTestEnvironmentId } from "../lib/adapter-test-environment";import { extractModelName, extractProviderId } from "../lib/model-utils";import { queryKeys } from "../lib/queryKeys";import { useCompany } from "../context/CompanyContext";@@ -662,7 +663,37 @@ export function AgentConfigForm(props: AgentConfigFormProps) {const adapterConfigPatch = flushedEnv ? { env: flushedEnv } : undefined;const primaryModel = currentModelId.trim() || null;const cheapTestCase = getCheapModelTestCase(adapterConfigPatch);- const environmentId = currentDefaultEnvironmentId || null;+ // Probe where a real run would actually execute: the agent's own+ // environment, else the instance default. Testing the host for an+ // agent that runs in the instance-default sandbox reports failures+ // (e.g. a CLI that only exists in the sandbox image) a real run would+ // never hit. The raw id is sent even for a local environment — the+ // server resolves the driver and probes the host in that case.+ //+ // Test can be clicked before the settings query settles (or after it+ // failed with retry:false), so when the agent relies on the instance+ // default, resolve the settings here rather than trusting the+ // render-time cache. A fetch that still fails FAILS the test with an+ // honest diagnostic — silently probing the host instead would report+ // the exact false command-not-found failure this resolution exists to+ // fix. Agents with their own environment never need the settings.+ let settings = instanceSettings;+ if (!rawCurrentDefaultEnvironmentId && settings === undefined) {+ try {+ settings = await queryClient.ensureQueryData({+ queryKey: queryKeys.instance.settings,+ queryFn: () => instanceSettingsApi.get(),+ });+ } catch {+ throw new Error(+ "Could not load instance settings to determine which environment to test in. Retry the test.",+ );+ }+ }+ const environmentId = resolveAdapterTestEnvironmentId({+ agentDefaultEnvironmentId: rawCurrentDefaultEnvironmentId || null,+ instanceDefaultEnvironmentId: settings?.defaultEnvironmentId ?? null,+ });const testResults: Array<{ label: string; model: string | null; result: AdapterEnvironmentTestResult }> = [{label: "Primary model",diff --git a/ui/src/lib/adapter-test-environment.test.ts b/ui/src/lib/adapter-test-environment.test.tsnew file mode 100644index 0000000000..856d085ced--- /dev/null+++ b/ui/src/lib/adapter-test-environment.test.ts@@ -0,0 +1,48 @@+import { describe, expect, it } from "vitest";++import { resolveAdapterTestEnvironmentId } from "./adapter-test-environment";++describe("resolveAdapterTestEnvironmentId", () => {+ it("prefers the agent's own environment", () => {+ expect(+ resolveAdapterTestEnvironmentId({+ agentDefaultEnvironmentId: "agent-env",+ instanceDefaultEnvironmentId: "instance-env",+ }),+ ).toBe("agent-env");+ });++ it("falls back to the instance default when the agent has none", () => {+ // The regression this pins: an agent relying on the instance default+ // (e.g. a managed sandbox with extra CLIs baked into its image) must be+ // tested inside that environment, not on the Paperclip host where the+ // CLI does not exist.+ expect(+ resolveAdapterTestEnvironmentId({+ agentDefaultEnvironmentId: null,+ instanceDefaultEnvironmentId: "instance-env",+ }),+ ).toBe("instance-env");+ expect(+ resolveAdapterTestEnvironmentId({+ agentDefaultEnvironmentId: "",+ instanceDefaultEnvironmentId: "instance-env",+ }),+ ).toBe("instance-env");+ });++ it("returns null (host probe) when neither is set", () => {+ expect(+ resolveAdapterTestEnvironmentId({+ agentDefaultEnvironmentId: undefined,+ instanceDefaultEnvironmentId: undefined,+ }),+ ).toBeNull();+ expect(+ resolveAdapterTestEnvironmentId({+ agentDefaultEnvironmentId: "",+ instanceDefaultEnvironmentId: null,+ }),+ ).toBeNull();+ });+});diff --git a/ui/src/lib/adapter-test-environment.ts b/ui/src/lib/adapter-test-environment.tsnew file mode 100644index 0000000000..541f320c57--- /dev/null+++ b/ui/src/lib/adapter-test-environment.ts@@ -0,0 +1,18 @@+/**+ * Which environment should an adapter "Test" probe?+ *+ * Mirrors the server's run-time resolution+ * (`resolveExecutionWorkspaceEnvironmentId`): the agent's own environment+ * wins, otherwise the instance default, otherwise none (the server probes+ * the Paperclip host). Without the instance-default fallback, the Test+ * button probes the host for agents that rely on the instance default and+ * fails on commands that only exist inside the default environment — for+ * example a sandbox image with an extra CLI installed — even though a real+ * run would have resolved to that environment and succeeded.+ */+export function resolveAdapterTestEnvironmentId(input: {+ agentDefaultEnvironmentId: string | null | undefined;+ instanceDefaultEnvironmentId: string | null | undefined;+}): string | null {+ return input.agentDefaultEnvironmentId || input.instanceDefaultEnvironmentId || null;+}