π OpenHands: AI-Driven Development
test(e2e): add mock-LLM coverage for litellm_proxy base_url preservation (#1183)
Rohit Malhotrae32608b60970
diff --git a/tests/e2e/mock-llm/mock-llm-profile-management.spec.ts b/tests/e2e/mock-llm/mock-llm-profile-management.spec.tsindex e6a7e9f503..9f1556e9a4 100644--- a/tests/e2e/mock-llm/mock-llm-profile-management.spec.ts+++ b/tests/e2e/mock-llm/mock-llm-profile-management.spec.ts@@ -1,7 +1,7 @@/*** Mock-LLM E2E tests: LLM profile management regressions.*- * Covers two scenarios that previously had no end-to-end guard:+ * Covers three scenarios that previously had no end-to-end guard:** 1. Active profile deletion + reconciliation:* The active LLM profile IS deletable (the PR #1127 disable-guard was@@ -16,6 +16,12 @@* user selected, not the first alphabetical match. The fix* stamps the active profile name on client-side conversation* metadata at creation and on per-conversation switches.+ *+ * 3. Proxy base_url preservation for litellm_proxy profiles (PR #1148):+ * When the SDK rewrites an `openhands/*` model to `litellm_proxy/*`+ * with the All-Hands proxy base_url, re-saving the profile from the+ * Basic tab must not strip the base_url. Without it the profile is+ * stranded and LiteLLM reroutes requests to the wrong endpoint.*/import { test, expect, type APIRequestContext } from "@playwright/test";@@ -47,6 +53,7 @@ async function saveProfile(request: APIRequestContext,name: string,model: string,+ baseUrl: string = MOCK_LLM_AGENT_URL,) {await request.delete(`${BACKEND_URL}/api/profiles/${encodeURIComponent(name)}`,@@ -63,7 +70,7 @@ async function saveProfile(llm: {model,api_key: "mock-api-key-for-testing",- base_url: MOCK_LLM_AGENT_URL,+ base_url: baseUrl,},include_secrets: true,},@@ -72,6 +79,24 @@ async function saveProfile(expect(resp.ok(), `POST /api/profiles/${name}: ${resp.status()}`).toBe(true);}+async function getProfileConfig(+ request: APIRequestContext,+ name: string,+): Promise<Record<string, unknown>> {+ const resp = await request.get(+ `${BACKEND_URL}/api/profiles/${encodeURIComponent(name)}`,+ {+ headers: {+ "X-Session-API-Key": SESSION_API_KEY,+ "X-Expose-Secrets": "encrypted",+ },+ },+ );+ expect(resp.ok(), `GET /api/profiles/${name}: ${resp.status()}`).toBe(true);+ const data = await resp.json();+ return (data.config ?? {}) as Record<string, unknown>;+}+async function activateProfile(request: APIRequestContext, name: string) {const resp = await request.post(`${BACKEND_URL}/api/profiles/${encodeURIComponent(name)}/activate`,@@ -349,3 +374,130 @@ test.describe("same-model profile identity", () => {});});});++// βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ+// Test 3 β Proxy base_url preservation for litellm_proxy profiles+// (PR #1148, issue #1146)+// βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ++test.describe("litellm_proxy proxy base_url preservation", () => {+ // Simulates the state the SDK persists after onboarding through the+ // OpenHands provider: openhands/* is rewritten to litellm_proxy/* and+ // paired with the All-Hands proxy base URL.+ const PROXY_PROFILE = "proxy-base-url-test";+ const LITELLM_PROXY_MODEL = "litellm_proxy/claude-opus-4-8";+ const OPENHANDS_PROXY_BASE_URL = "https://llm-proxy.app.all-hands.dev/";++ test.beforeEach(async ({ page }) => {+ await seedLocalStorage(page);+ });++ test.afterAll(async ({ request }) => {+ try {+ await deleteProfile(request, PROXY_PROFILE);+ } catch {+ // best-effort+ }+ });++ test("re-saving a litellm_proxy profile from Basic view preserves the proxy base_url", async ({+ page,+ request,+ }) => {+ // ββ Setup: create a profile with the SDK-rewritten litellm_proxy+ // model + proxy base_url, exactly as the agent-server persists it+ // after an openhands/* model selection during onboarding. ββ+ await saveProfile(+ request,+ PROXY_PROFILE,+ LITELLM_PROXY_MODEL,+ OPENHANDS_PROXY_BASE_URL,+ );++ // ββ Pre-check: verify the profile's base_url via API before the+ // UI round-trip so we know the starting state is correct. ββ+ await test.step("verify initial profile has proxy base_url", async () => {+ const config = await getProfileConfig(request, PROXY_PROFILE);+ expect(config.model).toBe(LITELLM_PROXY_MODEL);+ expect(config.base_url).toBe(OPENHANDS_PROXY_BASE_URL);+ });++ // ββ Navigate to the LLM settings page ββ+ await routeSessionApiKey(page);+ await page.goto("/settings/llm", { waitUntil: "domcontentloaded" });+ await dismissAnalyticsModal(page);+ await waitForTestId(page, "add-llm-profile");++ // ββ Find the profile row and open edit mode ββ+ await test.step("open profile in edit mode", async () => {+ const profileRows = page.getByTestId("profile-row");+ const rowCount = await profileRows.count();+ let targetRow: ReturnType<typeof profileRows.nth> | null = null;++ for (let i = 0; i < rowCount; i++) {+ const row = profileRows.nth(i);+ const text = await row.textContent();+ if (text?.includes(PROXY_PROFILE)) {+ targetRow = row;+ break;+ }+ }+ expect(+ targetRow,+ `Could not find profile row for "${PROXY_PROFILE}"`,+ ).not.toBeNull();++ await targetRow!.getByTestId("profile-menu-trigger").click();+ await waitForTestId(page, "profile-actions-menu");+ await page.getByTestId("profile-edit").click();++ // Wait for the editor to load with the profile data+ await expect(page.getByTestId("profile-name-input")).toHaveValue(+ PROXY_PROFILE,+ { timeout: 10_000 },+ );+ });++ // ββ Ensure we are on the Basic tab, then save ββ+ await test.step("switch to Basic view and save", async () => {+ // The litellm_proxy provider + proxy base_url is recognized as a+ // known default, so the form may already be on Basic. Click the+ // toggle to be explicit β this is the same user action the unit+ // test for PR #1148 exercises.+ const basicToggle = page.getByTestId("sdk-section-basic-toggle");+ if (await basicToggle.isVisible().catch(() => false)) {+ await basicToggle.click();+ }++ const saveButton = page.getByTestId("save-profile-btn");+ await expect(saveButton).toBeEnabled({ timeout: 10_000 });+ await saveButton.click();++ // Wait for save to complete β the UI navigates back to the+ // profile list after a successful save.+ await waitForTestId(page, "add-llm-profile");+ });++ // ββ Verify: the proxy base_url survived the Basic-tab save ββ+ await test.step("verify base_url is preserved after save", async () => {+ const config = await getProfileConfig(request, PROXY_PROFILE);+ expect(+ config.base_url,+ "base_url must be the All-Hands proxy URL after a Basic-tab re-save; " ++ "dropping it strands the profile (issue #1146)",+ ).toBe(OPENHANDS_PROXY_BASE_URL);+ expect(config.model).toBe(LITELLM_PROXY_MODEL);+ });++ // ββ Verify: the profile also looks correct after a page reload ββ+ await test.step("profile survives page reload", async () => {+ await page.reload({ waitUntil: "domcontentloaded" });+ await waitForTestId(page, "add-llm-profile");++ // Re-read via API to confirm persistence is durable+ const config = await getProfileConfig(request, PROXY_PROFILE);+ expect(config.base_url).toBe(OPENHANDS_PROXY_BASE_URL);+ expect(config.model).toBe(LITELLM_PROXY_MODEL);+ });+ });+});