Skip to content

The open-source app everyone uses to manage agents at work

Test cheap model during agent config check (#8632)

Devin Foley5170a9d35d00
diff --git a/ui/src/components/AgentConfigForm.render.test.tsx b/ui/src/components/AgentConfigForm.render.test.tsx
index 9d94ed5073..f28a1d826a 100644
--- a/ui/src/components/AgentConfigForm.render.test.tsx
+++ b/ui/src/components/AgentConfigForm.render.test.tsx
@@ -9,9 +9,11 @@ import { TooltipProvider } from "@/components/ui/tooltip";
import { AgentConfigForm } from "./AgentConfigForm";
const mockAgentsApi = vi.hoisted(() => ({
+ adapterModelProfiles: vi.fn(),
adapterModels: vi.fn(),
detectModel: vi.fn(),
list: vi.fn(),
+ testEnvironment: vi.fn(),
}));
const mockEnvironmentsApi = vi.hoisted(() => ({
@@ -155,7 +157,11 @@ function makeEnvironment(overrides: Partial<Environment>): Environment {
};
}
-async function renderForm(environments: Environment[], agentOverrides: Partial<Agent> = {}) {
+async function renderForm(
+ environments: Environment[],
+ agentOverrides: Partial<Agent> = {},
+ options: { showAdapterTestEnvironmentButton?: boolean } = {},
+) {
mockEnvironmentsApi.list.mockResolvedValue(environments);
const container = document.createElement("div");
@@ -178,7 +184,7 @@ async function renderForm(environments: Environment[], agentOverrides: Partial<A
onSave={vi.fn()}
hidePromptTemplate
showAdapterTypeField={false}
- showAdapterTestEnvironmentButton={false}
+ showAdapterTestEnvironmentButton={options.showAdapterTestEnvironmentButton ?? false}
/>
</TooltipProvider>
</QueryClientProvider>,
@@ -193,9 +199,16 @@ describe("AgentConfigForm environment selector", () => {
let roots: Root[] = [];
beforeEach(() => {
+ mockAgentsApi.adapterModelProfiles.mockResolvedValue([]);
mockAgentsApi.adapterModels.mockResolvedValue([]);
mockAgentsApi.detectModel.mockResolvedValue(null);
mockAgentsApi.list.mockResolvedValue([]);
+ mockAgentsApi.testEnvironment.mockResolvedValue({
+ adapterType: "codex_local",
+ status: "pass",
+ checks: [],
+ testedAt: new Date(0).toISOString(),
+ });
mockInstanceSettingsApi.get.mockResolvedValue({ defaultEnvironmentId: null });
mockInstanceSettingsApi.getExperimental.mockResolvedValue({ enableEnvironments: true });
mockInstanceSettingsApi.getGeneral.mockResolvedValue({ executionMode: "any" });
@@ -269,4 +282,83 @@ describe("AgentConfigForm environment selector", () => {
expect(selector?.textContent).toContain("Default: Local");
expect(selector?.textContent).toContain("Fake Sandbox · sandbox");
});
+
+ it("tests both the primary and cheap models when a cheap profile is configured", async () => {
+ const result = await renderForm([
+ makeEnvironment({ id: "local-1", name: "Local", driver: "local" }),
+ ], {
+ adapterConfig: { model: "gpt-5.4" },
+ runtimeConfig: {
+ modelProfiles: {
+ cheap: {
+ enabled: true,
+ adapterConfig: {
+ model: "gpt-5.4-mini",
+ baseUrl: "https://cheap-models.example.test",
+ provider: "budget-provider",
+ },
+ },
+ },
+ },
+ }, {
+ showAdapterTestEnvironmentButton: true,
+ });
+ roots.push(result.root);
+
+ const testButton = Array.from(result.container.querySelectorAll("button")).find(
+ (button) => button.textContent?.trim() === "Test",
+ );
+ expect(testButton).toBeTruthy();
+
+ await act(async () => {
+ testButton?.dispatchEvent(new MouseEvent("click", { bubbles: true }));
+ });
+ await flushReact();
+
+ expect(mockAgentsApi.testEnvironment).toHaveBeenCalledTimes(2);
+ expect(mockAgentsApi.testEnvironment.mock.calls[0]?.[2]).toMatchObject({
+ adapterConfig: expect.objectContaining({ model: "gpt-5.4" }),
+ });
+ expect(mockAgentsApi.testEnvironment.mock.calls[1]?.[2]).toMatchObject({
+ adapterConfig: expect.objectContaining({
+ model: "gpt-5.4-mini",
+ baseUrl: "https://cheap-models.example.test",
+ provider: "budget-provider",
+ }),
+ });
+ });
+
+ it("surfaces request failures instead of converting them into model test checks", async () => {
+ mockAgentsApi.testEnvironment.mockRejectedValueOnce(new Error("Network unavailable"));
+
+ const result = await renderForm([
+ makeEnvironment({ id: "local-1", name: "Local", driver: "local" }),
+ ], {
+ adapterConfig: { model: "gpt-5.4" },
+ runtimeConfig: {
+ modelProfiles: {
+ cheap: {
+ enabled: true,
+ adapterConfig: { model: "gpt-5.4-mini" },
+ },
+ },
+ },
+ }, {
+ showAdapterTestEnvironmentButton: true,
+ });
+ roots.push(result.root);
+
+ const testButton = Array.from(result.container.querySelectorAll("button")).find(
+ (button) => button.textContent?.trim() === "Test",
+ );
+ expect(testButton).toBeTruthy();
+
+ await act(async () => {
+ testButton?.dispatchEvent(new MouseEvent("click", { bubbles: true }));
+ });
+ await flushReact();
+
+ expect(mockAgentsApi.testEnvironment).toHaveBeenCalledTimes(1);
+ expect(result.container.textContent).toContain("Network unavailable");
+ });
});
diff --git a/ui/src/components/AgentConfigForm.tsx b/ui/src/components/AgentConfigForm.tsx
index a673f2ca07..06f12521e9 100644
--- a/ui/src/components/AgentConfigForm.tsx
+++ b/ui/src/components/AgentConfigForm.tsx
@@ -516,12 +516,19 @@ export function AgentConfigForm(props: AgentConfigFormProps) {
return typeof value === "string" ? value : "";
}, [adapterCheapDefault]);
- function buildAdapterConfigForTest(): Record<string, unknown> {
+ function buildAdapterConfigForTest(adapterConfigPatch?: Record<string, unknown>): Record<string, unknown> {
if (isCreate) {
- return uiAdapter.buildAdapterConfig(val!);
+ const next = uiAdapter.buildAdapterConfig(val!);
+ if (adapterConfigPatch) {
+ Object.assign(next, adapterConfigPatch);
+ }
+ return next;
}
const base = config as Record<string, unknown>;
const next = { ...base, ...overlay.adapterConfig };
+ if (adapterConfigPatch) {
+ Object.assign(next, adapterConfigPatch);
+ }
if (adapterType === "hermes_local") {
const hermesCommand =
typeof next.hermesCommand === "string" && next.hermesCommand.length > 0
@@ -536,15 +543,123 @@ export function AgentConfigForm(props: AgentConfigFormProps) {
return next;
}
+ function buildCheapAdapterConfigForTest(): Record<string, unknown> {
+ const adapterDefaultConfig = asObject(adapterCheapDefault?.adapterConfig);
+ const createCheapModel = isCreate ? (val!.cheapModel ?? "").trim() : "";
+ const cheapAdapterConfig = isCreate
+ ? {
+ ...adapterDefaultConfig,
+ ...(createCheapModel ? { model: createCheapModel } : {}),
+ }
+ : {
+ ...adapterDefaultConfig,
+ ...cheapProfileFromAgent.adapterConfig,
+ ...asObject(cheapOverlay?.adapterConfig),
+ };
+ return buildAdapterConfigForTest(cheapAdapterConfig);
+ }
+
+ function getCheapModelTestCase(): { model: string; adapterConfig: Record<string, unknown> } | null {
+ if (!currentCheapEnabled) return null;
+ const adapterConfig = buildCheapAdapterConfigForTest();
+ const configModel = typeof adapterConfig.model === "string" ? adapterConfig.model.trim() : "";
+ const model = configModel || currentCheapModel.trim();
+ if (!model) return null;
+ adapterConfig.model = model;
+ return { model, adapterConfig };
+ }
+
+ function prefixEnvironmentTestChecks(
+ result: AdapterEnvironmentTestResult,
+ label: string,
+ model: string | null,
+ ): AdapterEnvironmentTestResult {
+ const modelLabel = model ? ` (${model})` : "";
+ return {
+ ...result,
+ checks: [
+ {
+ code: `${label.toLowerCase().replace(/[^a-z0-9]+/g, "_")}_test_started`,
+ level: "info",
+ message: `${label} test${modelLabel}`,
+ },
+ ...result.checks.map((check) => ({
+ ...check,
+ message: `${label} test${modelLabel}: ${check.message}`,
+ })),
+ ],
+ };
+ }
+
+ async function runEnvironmentTestCase(
+ label: string,
+ model: string | null,
+ adapterConfig: Record<string, unknown>,
+ environmentId: string | null,
+ ): Promise<AdapterEnvironmentTestResult> {
+ const result = await agentsApi.testEnvironment(selectedCompanyId!, adapterType, {
+ adapterConfig,
+ environmentId,
+ });
+ return prefixEnvironmentTestChecks(result, label, model);
+ }
+
+ function mergeEnvironmentTestResults(
+ results: AdapterEnvironmentTestResult[],
+ ): AdapterEnvironmentTestResult {
+ const checks = results.flatMap((result) => result.checks);
+ const status = results.some((result) => result.status === "fail")
+ ? "fail"
+ : results.some((result) => result.status === "warn")
+ ? "warn"
+ : "pass";
+ const testedAt = results[results.length - 1]?.testedAt ?? new Date().toISOString();
+
+ return {
+ adapterType,
+ status,
+ checks,
+ testedAt,
+ };
+ }
+
const testEnvironment = useMutation({
mutationFn: async () => {
if (!selectedCompanyId) {
throw new Error("Select a company to test adapter environment");
}
- return agentsApi.testEnvironment(selectedCompanyId, adapterType, {
- adapterConfig: buildAdapterConfigForTest(),
- environmentId: currentDefaultEnvironmentId || null,
- });
+ const primaryModel = currentModelId.trim() || null;
+ const cheapTestCase = getCheapModelTestCase();
+ const environmentId = currentDefaultEnvironmentId || null;
+ const testResults: Array<{ label: string; model: string | null; result: AdapterEnvironmentTestResult }> = [
+ {
+ label: "Primary model",
+ model: primaryModel,
+ result: await runEnvironmentTestCase(
+ "Primary model",
+ primaryModel,
+ buildAdapterConfigForTest(),
+ environmentId,
+ ),
+ },
+ ];
+
+ if (cheapTestCase) {
+ testResults.push({
+ label: "Cheap model",
+ model: cheapTestCase.model,
+ result: await runEnvironmentTestCase(
+ "Cheap model",
+ cheapTestCase.model,
+ cheapTestCase.adapterConfig,
+ environmentId,
+ ),
+ });
+ }
+
+ return testResults.length > 1
+ ? mergeEnvironmentTestResults(testResults.map(({ result }) => result))
+ : testResults[0]!.result;
},
});
const [testActionPending, setTestActionPending] = useState(false);
@@ -692,9 +807,10 @@ export function AgentConfigForm(props: AgentConfigFormProps) {
const cheapProfileFromAgent = useMemo(() => {
const profiles = (runtimeConfig.modelProfiles ?? {}) as Record<string, unknown>;
const cheap = (profiles.cheap ?? {}) as Record<string, unknown>;
- const cheapAdapterConfig = (cheap.adapterConfig ?? {}) as Record<string, unknown>;
+ const cheapAdapterConfig = asObject(cheap.adapterConfig);
return {
enabled: cheap.enabled !== false,
+ adapterConfig: cheapAdapterConfig,
model: typeof cheapAdapterConfig.model === "string" ? cheapAdapterConfig.model : "",
};
}, [runtimeConfig]);