Skip to content

The open-source app everyone uses to manage agents at work

Test adapters in the environment a run would actually use (#10698)

Devin Foley2ffebd48363b
diff --git a/ui/src/components/AgentConfigForm.tsx b/ui/src/components/AgentConfigForm.tsx
index bbaa9cf67f..18bbaff7df 100644
--- a/ui/src/components/AgentConfigForm.tsx
+++ b/ui/src/components/AgentConfigForm.tsx
@@ -27,6 +27,7 @@ import {
import { Button } from "@/components/ui/button";
import { FolderOpen, Heart, ChevronDown, X } from "lucide-react";
import { asBoolean, asFiniteNumber, asObject, cn } from "../lib/utils";
+import { resolveAdapterTestEnvironmentId } from "../lib/adapter-test-environment";
import { extractModelName, extractProviderId } from "../lib/model-utils";
import { queryKeys } from "../lib/queryKeys";
import { useCompany } from "../context/CompanyContext";
@@ -662,7 +663,37 @@ export function AgentConfigForm(props: AgentConfigFormProps) {
const adapterConfigPatch = flushedEnv ? { env: flushedEnv } : undefined;
const primaryModel = currentModelId.trim() || null;
const cheapTestCase = getCheapModelTestCase(adapterConfigPatch);
- const environmentId = currentDefaultEnvironmentId || null;
+ // Probe where a real run would actually execute: the agent's own
+ // environment, else the instance default. Testing the host for an
+ // agent that runs in the instance-default sandbox reports failures
+ // (e.g. a CLI that only exists in the sandbox image) a real run would
+ // never hit. The raw id is sent even for a local environment — the
+ // server resolves the driver and probes the host in that case.
+ //
+ // Test can be clicked before the settings query settles (or after it
+ // failed with retry:false), so when the agent relies on the instance
+ // default, resolve the settings here rather than trusting the
+ // render-time cache. A fetch that still fails FAILS the test with an
+ // honest diagnostic — silently probing the host instead would report
+ // the exact false command-not-found failure this resolution exists to
+ // fix. Agents with their own environment never need the settings.
+ let settings = instanceSettings;
+ if (!rawCurrentDefaultEnvironmentId && settings === undefined) {
+ try {
+ settings = await queryClient.ensureQueryData({
+ queryKey: queryKeys.instance.settings,
+ queryFn: () => instanceSettingsApi.get(),
+ });
+ } catch {
+ throw new Error(
+ "Could not load instance settings to determine which environment to test in. Retry the test.",
+ );
+ }
+ }
+ const environmentId = resolveAdapterTestEnvironmentId({
+ agentDefaultEnvironmentId: rawCurrentDefaultEnvironmentId || null,
+ instanceDefaultEnvironmentId: settings?.defaultEnvironmentId ?? null,
+ });
const testResults: Array<{ label: string; model: string | null; result: AdapterEnvironmentTestResult }> = [
{
label: "Primary model",
diff --git a/ui/src/lib/adapter-test-environment.test.ts b/ui/src/lib/adapter-test-environment.test.ts
new file mode 100644
index 0000000000..856d085ced
--- /dev/null
+++ b/ui/src/lib/adapter-test-environment.test.ts
@@ -0,0 +1,48 @@
+import { describe, expect, it } from "vitest";
+
+import { resolveAdapterTestEnvironmentId } from "./adapter-test-environment";
+
+describe("resolveAdapterTestEnvironmentId", () => {
+ it("prefers the agent's own environment", () => {
+ expect(
+ resolveAdapterTestEnvironmentId({
+ agentDefaultEnvironmentId: "agent-env",
+ instanceDefaultEnvironmentId: "instance-env",
+ }),
+ ).toBe("agent-env");
+ });
+
+ it("falls back to the instance default when the agent has none", () => {
+ // The regression this pins: an agent relying on the instance default
+ // (e.g. a managed sandbox with extra CLIs baked into its image) must be
+ // tested inside that environment, not on the Paperclip host where the
+ // CLI does not exist.
+ expect(
+ resolveAdapterTestEnvironmentId({
+ agentDefaultEnvironmentId: null,
+ instanceDefaultEnvironmentId: "instance-env",
+ }),
+ ).toBe("instance-env");
+ expect(
+ resolveAdapterTestEnvironmentId({
+ agentDefaultEnvironmentId: "",
+ instanceDefaultEnvironmentId: "instance-env",
+ }),
+ ).toBe("instance-env");
+ });
+
+ it("returns null (host probe) when neither is set", () => {
+ expect(
+ resolveAdapterTestEnvironmentId({
+ agentDefaultEnvironmentId: undefined,
+ instanceDefaultEnvironmentId: undefined,
+ }),
+ ).toBeNull();
+ expect(
+ resolveAdapterTestEnvironmentId({
+ agentDefaultEnvironmentId: "",
+ instanceDefaultEnvironmentId: null,
+ }),
+ ).toBeNull();
+ });
+});
diff --git a/ui/src/lib/adapter-test-environment.ts b/ui/src/lib/adapter-test-environment.ts
new file mode 100644
index 0000000000..541f320c57
--- /dev/null
+++ b/ui/src/lib/adapter-test-environment.ts
@@ -0,0 +1,18 @@
+/**
+ * Which environment should an adapter "Test" probe?
+ *
+ * Mirrors the server's run-time resolution
+ * (`resolveExecutionWorkspaceEnvironmentId`): the agent's own environment
+ * wins, otherwise the instance default, otherwise none (the server probes
+ * the Paperclip host). Without the instance-default fallback, the Test
+ * button probes the host for agents that rely on the instance default and
+ * fails on commands that only exist inside the default environment — for
+ * example a sandbox image with an extra CLI installed — even though a real
+ * run would have resolved to that environment and succeeded.
+ */
+export function resolveAdapterTestEnvironmentId(input: {
+ agentDefaultEnvironmentId: string | null | undefined;
+ instanceDefaultEnvironmentId: string | null | undefined;
+}): string | null {
+ return input.agentDefaultEnvironmentId || input.instanceDefaultEnvironmentId || null;
+}