The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
Test cheap model during agent config check (#8632)
Devin Foley5170a9d35d00
diff --git a/ui/src/components/AgentConfigForm.render.test.tsx b/ui/src/components/AgentConfigForm.render.test.tsxindex 9d94ed5073..f28a1d826a 100644--- a/ui/src/components/AgentConfigForm.render.test.tsx+++ b/ui/src/components/AgentConfigForm.render.test.tsx@@ -9,9 +9,11 @@ import { TooltipProvider } from "@/components/ui/tooltip";import { AgentConfigForm } from "./AgentConfigForm";const mockAgentsApi = vi.hoisted(() => ({+ adapterModelProfiles: vi.fn(),adapterModels: vi.fn(),detectModel: vi.fn(),list: vi.fn(),+ testEnvironment: vi.fn(),}));const mockEnvironmentsApi = vi.hoisted(() => ({@@ -155,7 +157,11 @@ function makeEnvironment(overrides: Partial<Environment>): Environment {};}-async function renderForm(environments: Environment[], agentOverrides: Partial<Agent> = {}) {+async function renderForm(+ environments: Environment[],+ agentOverrides: Partial<Agent> = {},+ options: { showAdapterTestEnvironmentButton?: boolean } = {},+) {mockEnvironmentsApi.list.mockResolvedValue(environments);const container = document.createElement("div");@@ -178,7 +184,7 @@ async function renderForm(environments: Environment[], agentOverrides: Partial<AonSave={vi.fn()}hidePromptTemplateshowAdapterTypeField={false}- showAdapterTestEnvironmentButton={false}+ showAdapterTestEnvironmentButton={options.showAdapterTestEnvironmentButton ?? false}/></TooltipProvider></QueryClientProvider>,@@ -193,9 +199,16 @@ describe("AgentConfigForm environment selector", () => {let roots: Root[] = [];beforeEach(() => {+ mockAgentsApi.adapterModelProfiles.mockResolvedValue([]);mockAgentsApi.adapterModels.mockResolvedValue([]);mockAgentsApi.detectModel.mockResolvedValue(null);mockAgentsApi.list.mockResolvedValue([]);+ mockAgentsApi.testEnvironment.mockResolvedValue({+ adapterType: "codex_local",+ status: "pass",+ checks: [],+ testedAt: new Date(0).toISOString(),+ });mockInstanceSettingsApi.get.mockResolvedValue({ defaultEnvironmentId: null });mockInstanceSettingsApi.getExperimental.mockResolvedValue({ enableEnvironments: true });mockInstanceSettingsApi.getGeneral.mockResolvedValue({ executionMode: "any" });@@ -269,4 +282,83 @@ describe("AgentConfigForm environment selector", () => {expect(selector?.textContent).toContain("Default: Local");expect(selector?.textContent).toContain("Fake Sandbox · sandbox");});++ it("tests both the primary and cheap models when a cheap profile is configured", async () => {+ const result = await renderForm([+ makeEnvironment({ id: "local-1", name: "Local", driver: "local" }),+ ], {+ adapterConfig: { model: "gpt-5.4" },+ runtimeConfig: {+ modelProfiles: {+ cheap: {+ enabled: true,+ adapterConfig: {+ model: "gpt-5.4-mini",+ baseUrl: "https://cheap-models.example.test",+ provider: "budget-provider",+ },+ },+ },+ },+ }, {+ showAdapterTestEnvironmentButton: true,+ });+ roots.push(result.root);++ const testButton = Array.from(result.container.querySelectorAll("button")).find(+ (button) => button.textContent?.trim() === "Test",+ );+ expect(testButton).toBeTruthy();++ await act(async () => {+ testButton?.dispatchEvent(new MouseEvent("click", { bubbles: true }));+ });+ await flushReact();++ expect(mockAgentsApi.testEnvironment).toHaveBeenCalledTimes(2);+ expect(mockAgentsApi.testEnvironment.mock.calls[0]?.[2]).toMatchObject({+ adapterConfig: expect.objectContaining({ model: "gpt-5.4" }),+ });+ expect(mockAgentsApi.testEnvironment.mock.calls[1]?.[2]).toMatchObject({+ adapterConfig: expect.objectContaining({+ model: "gpt-5.4-mini",+ baseUrl: "https://cheap-models.example.test",+ provider: "budget-provider",+ }),+ });+ });++ it("surfaces request failures instead of converting them into model test checks", async () => {+ mockAgentsApi.testEnvironment.mockRejectedValueOnce(new Error("Network unavailable"));++ const result = await renderForm([+ makeEnvironment({ id: "local-1", name: "Local", driver: "local" }),+ ], {+ adapterConfig: { model: "gpt-5.4" },+ runtimeConfig: {+ modelProfiles: {+ cheap: {+ enabled: true,+ adapterConfig: { model: "gpt-5.4-mini" },+ },+ },+ },+ }, {+ showAdapterTestEnvironmentButton: true,+ });+ roots.push(result.root);++ const testButton = Array.from(result.container.querySelectorAll("button")).find(+ (button) => button.textContent?.trim() === "Test",+ );+ expect(testButton).toBeTruthy();++ await act(async () => {+ testButton?.dispatchEvent(new MouseEvent("click", { bubbles: true }));+ });+ await flushReact();++ expect(mockAgentsApi.testEnvironment).toHaveBeenCalledTimes(1);+ expect(result.container.textContent).toContain("Network unavailable");+ });});diff --git a/ui/src/components/AgentConfigForm.tsx b/ui/src/components/AgentConfigForm.tsxindex a673f2ca07..06f12521e9 100644--- a/ui/src/components/AgentConfigForm.tsx+++ b/ui/src/components/AgentConfigForm.tsx@@ -516,12 +516,19 @@ export function AgentConfigForm(props: AgentConfigFormProps) {return typeof value === "string" ? value : "";}, [adapterCheapDefault]);- function buildAdapterConfigForTest(): Record<string, unknown> {+ function buildAdapterConfigForTest(adapterConfigPatch?: Record<string, unknown>): Record<string, unknown> {if (isCreate) {- return uiAdapter.buildAdapterConfig(val!);+ const next = uiAdapter.buildAdapterConfig(val!);+ if (adapterConfigPatch) {+ Object.assign(next, adapterConfigPatch);+ }+ return next;}const base = config as Record<string, unknown>;const next = { ...base, ...overlay.adapterConfig };+ if (adapterConfigPatch) {+ Object.assign(next, adapterConfigPatch);+ }if (adapterType === "hermes_local") {const hermesCommand =typeof next.hermesCommand === "string" && next.hermesCommand.length > 0@@ -536,15 +543,123 @@ export function AgentConfigForm(props: AgentConfigFormProps) {return next;}+ function buildCheapAdapterConfigForTest(): Record<string, unknown> {+ const adapterDefaultConfig = asObject(adapterCheapDefault?.adapterConfig);+ const createCheapModel = isCreate ? (val!.cheapModel ?? "").trim() : "";+ const cheapAdapterConfig = isCreate+ ? {+ ...adapterDefaultConfig,+ ...(createCheapModel ? { model: createCheapModel } : {}),+ }+ : {+ ...adapterDefaultConfig,+ ...cheapProfileFromAgent.adapterConfig,+ ...asObject(cheapOverlay?.adapterConfig),+ };+ return buildAdapterConfigForTest(cheapAdapterConfig);+ }++ function getCheapModelTestCase(): { model: string; adapterConfig: Record<string, unknown> } | null {+ if (!currentCheapEnabled) return null;+ const adapterConfig = buildCheapAdapterConfigForTest();+ const configModel = typeof adapterConfig.model === "string" ? adapterConfig.model.trim() : "";+ const model = configModel || currentCheapModel.trim();+ if (!model) return null;+ adapterConfig.model = model;+ return { model, adapterConfig };+ }++ function prefixEnvironmentTestChecks(+ result: AdapterEnvironmentTestResult,+ label: string,+ model: string | null,+ ): AdapterEnvironmentTestResult {+ const modelLabel = model ? ` (${model})` : "";+ return {+ ...result,+ checks: [+ {+ code: `${label.toLowerCase().replace(/[^a-z0-9]+/g, "_")}_test_started`,+ level: "info",+ message: `${label} test${modelLabel}`,+ },+ ...result.checks.map((check) => ({+ ...check,+ message: `${label} test${modelLabel}: ${check.message}`,+ })),+ ],+ };+ }++ async function runEnvironmentTestCase(+ label: string,+ model: string | null,+ adapterConfig: Record<string, unknown>,+ environmentId: string | null,+ ): Promise<AdapterEnvironmentTestResult> {+ const result = await agentsApi.testEnvironment(selectedCompanyId!, adapterType, {+ adapterConfig,+ environmentId,+ });+ return prefixEnvironmentTestChecks(result, label, model);+ }++ function mergeEnvironmentTestResults(+ results: AdapterEnvironmentTestResult[],+ ): AdapterEnvironmentTestResult {+ const checks = results.flatMap((result) => result.checks);+ const status = results.some((result) => result.status === "fail")+ ? "fail"+ : results.some((result) => result.status === "warn")+ ? "warn"+ : "pass";+ const testedAt = results[results.length - 1]?.testedAt ?? new Date().toISOString();++ return {+ adapterType,+ status,+ checks,+ testedAt,+ };+ }+const testEnvironment = useMutation({mutationFn: async () => {if (!selectedCompanyId) {throw new Error("Select a company to test adapter environment");}- return agentsApi.testEnvironment(selectedCompanyId, adapterType, {- adapterConfig: buildAdapterConfigForTest(),- environmentId: currentDefaultEnvironmentId || null,- });+ const primaryModel = currentModelId.trim() || null;+ const cheapTestCase = getCheapModelTestCase();+ const environmentId = currentDefaultEnvironmentId || null;+ const testResults: Array<{ label: string; model: string | null; result: AdapterEnvironmentTestResult }> = [+ {+ label: "Primary model",+ model: primaryModel,+ result: await runEnvironmentTestCase(+ "Primary model",+ primaryModel,+ buildAdapterConfigForTest(),+ environmentId,+ ),+ },+ ];++ if (cheapTestCase) {+ testResults.push({+ label: "Cheap model",+ model: cheapTestCase.model,+ result: await runEnvironmentTestCase(+ "Cheap model",+ cheapTestCase.model,+ cheapTestCase.adapterConfig,+ environmentId,+ ),+ });+ }++ return testResults.length > 1+ ? mergeEnvironmentTestResults(testResults.map(({ result }) => result))+ : testResults[0]!.result;},});const [testActionPending, setTestActionPending] = useState(false);@@ -692,9 +807,10 @@ export function AgentConfigForm(props: AgentConfigFormProps) {const cheapProfileFromAgent = useMemo(() => {const profiles = (runtimeConfig.modelProfiles ?? {}) as Record<string, unknown>;const cheap = (profiles.cheap ?? {}) as Record<string, unknown>;- const cheapAdapterConfig = (cheap.adapterConfig ?? {}) as Record<string, unknown>;+ const cheapAdapterConfig = asObject(cheap.adapterConfig);return {enabled: cheap.enabled !== false,+ adapterConfig: cheapAdapterConfig,model: typeof cheapAdapterConfig.model === "string" ? cheapAdapterConfig.model : "",};}, [runtimeConfig]);