Skip to content

πŸ™Œ OpenHands: AI-Driven Development

test(e2e): add mock-LLM coverage for litellm_proxy base_url preservation (#1183)

Rohit Malhotrae32608b60970
diff --git a/tests/e2e/mock-llm/mock-llm-profile-management.spec.ts b/tests/e2e/mock-llm/mock-llm-profile-management.spec.ts
index e6a7e9f503..9f1556e9a4 100644
--- a/tests/e2e/mock-llm/mock-llm-profile-management.spec.ts
+++ b/tests/e2e/mock-llm/mock-llm-profile-management.spec.ts
@@ -1,7 +1,7 @@
/**
* Mock-LLM E2E tests: LLM profile management regressions.
*
- * Covers two scenarios that previously had no end-to-end guard:
+ * Covers three scenarios that previously had no end-to-end guard:
*
* 1. Active profile deletion + reconciliation:
* The active LLM profile IS deletable (the PR #1127 disable-guard was
@@ -16,6 +16,12 @@
* user selected, not the first alphabetical match. The fix
* stamps the active profile name on client-side conversation
* metadata at creation and on per-conversation switches.
+ *
+ * 3. Proxy base_url preservation for litellm_proxy profiles (PR #1148):
+ * When the SDK rewrites an `openhands/*` model to `litellm_proxy/*`
+ * with the All-Hands proxy base_url, re-saving the profile from the
+ * Basic tab must not strip the base_url. Without it the profile is
+ * stranded and LiteLLM reroutes requests to the wrong endpoint.
*/
import { test, expect, type APIRequestContext } from "@playwright/test";
@@ -47,6 +53,7 @@ async function saveProfile(
request: APIRequestContext,
name: string,
model: string,
+ baseUrl: string = MOCK_LLM_AGENT_URL,
) {
await request.delete(
`${BACKEND_URL}/api/profiles/${encodeURIComponent(name)}`,
@@ -63,7 +70,7 @@ async function saveProfile(
llm: {
model,
api_key: "mock-api-key-for-testing",
- base_url: MOCK_LLM_AGENT_URL,
+ base_url: baseUrl,
},
include_secrets: true,
},
@@ -72,6 +79,24 @@ async function saveProfile(
expect(resp.ok(), `POST /api/profiles/${name}: ${resp.status()}`).toBe(true);
}
+async function getProfileConfig(
+ request: APIRequestContext,
+ name: string,
+): Promise<Record<string, unknown>> {
+ const resp = await request.get(
+ `${BACKEND_URL}/api/profiles/${encodeURIComponent(name)}`,
+ {
+ headers: {
+ "X-Session-API-Key": SESSION_API_KEY,
+ "X-Expose-Secrets": "encrypted",
+ },
+ },
+ );
+ expect(resp.ok(), `GET /api/profiles/${name}: ${resp.status()}`).toBe(true);
+ const data = await resp.json();
+ return (data.config ?? {}) as Record<string, unknown>;
+}
+
async function activateProfile(request: APIRequestContext, name: string) {
const resp = await request.post(
`${BACKEND_URL}/api/profiles/${encodeURIComponent(name)}/activate`,
@@ -349,3 +374,130 @@ test.describe("same-model profile identity", () => {
});
});
});
+
+// ═══════════════════════════════════════════════════════════════════════
+// Test 3 β€” Proxy base_url preservation for litellm_proxy profiles
+// (PR #1148, issue #1146)
+// ═══════════════════════════════════════════════════════════════════════
+
+test.describe("litellm_proxy proxy base_url preservation", () => {
+ // Simulates the state the SDK persists after onboarding through the
+ // OpenHands provider: openhands/* is rewritten to litellm_proxy/* and
+ // paired with the All-Hands proxy base URL.
+ const PROXY_PROFILE = "proxy-base-url-test";
+ const LITELLM_PROXY_MODEL = "litellm_proxy/claude-opus-4-8";
+ const OPENHANDS_PROXY_BASE_URL = "https://llm-proxy.app.all-hands.dev/";
+
+ test.beforeEach(async ({ page }) => {
+ await seedLocalStorage(page);
+ });
+
+ test.afterAll(async ({ request }) => {
+ try {
+ await deleteProfile(request, PROXY_PROFILE);
+ } catch {
+ // best-effort
+ }
+ });
+
+ test("re-saving a litellm_proxy profile from Basic view preserves the proxy base_url", async ({
+ page,
+ request,
+ }) => {
+ // ── Setup: create a profile with the SDK-rewritten litellm_proxy
+ // model + proxy base_url, exactly as the agent-server persists it
+ // after an openhands/* model selection during onboarding. ──
+ await saveProfile(
+ request,
+ PROXY_PROFILE,
+ LITELLM_PROXY_MODEL,
+ OPENHANDS_PROXY_BASE_URL,
+ );
+
+ // ── Pre-check: verify the profile's base_url via API before the
+ // UI round-trip so we know the starting state is correct. ──
+ await test.step("verify initial profile has proxy base_url", async () => {
+ const config = await getProfileConfig(request, PROXY_PROFILE);
+ expect(config.model).toBe(LITELLM_PROXY_MODEL);
+ expect(config.base_url).toBe(OPENHANDS_PROXY_BASE_URL);
+ });
+
+ // ── Navigate to the LLM settings page ──
+ await routeSessionApiKey(page);
+ await page.goto("/settings/llm", { waitUntil: "domcontentloaded" });
+ await dismissAnalyticsModal(page);
+ await waitForTestId(page, "add-llm-profile");
+
+ // ── Find the profile row and open edit mode ──
+ await test.step("open profile in edit mode", async () => {
+ const profileRows = page.getByTestId("profile-row");
+ const rowCount = await profileRows.count();
+ let targetRow: ReturnType<typeof profileRows.nth> | null = null;
+
+ for (let i = 0; i < rowCount; i++) {
+ const row = profileRows.nth(i);
+ const text = await row.textContent();
+ if (text?.includes(PROXY_PROFILE)) {
+ targetRow = row;
+ break;
+ }
+ }
+ expect(
+ targetRow,
+ `Could not find profile row for "${PROXY_PROFILE}"`,
+ ).not.toBeNull();
+
+ await targetRow!.getByTestId("profile-menu-trigger").click();
+ await waitForTestId(page, "profile-actions-menu");
+ await page.getByTestId("profile-edit").click();
+
+ // Wait for the editor to load with the profile data
+ await expect(page.getByTestId("profile-name-input")).toHaveValue(
+ PROXY_PROFILE,
+ { timeout: 10_000 },
+ );
+ });
+
+ // ── Ensure we are on the Basic tab, then save ──
+ await test.step("switch to Basic view and save", async () => {
+ // The litellm_proxy provider + proxy base_url is recognized as a
+ // known default, so the form may already be on Basic. Click the
+ // toggle to be explicit β€” this is the same user action the unit
+ // test for PR #1148 exercises.
+ const basicToggle = page.getByTestId("sdk-section-basic-toggle");
+ if (await basicToggle.isVisible().catch(() => false)) {
+ await basicToggle.click();
+ }
+
+ const saveButton = page.getByTestId("save-profile-btn");
+ await expect(saveButton).toBeEnabled({ timeout: 10_000 });
+ await saveButton.click();
+
+ // Wait for save to complete β€” the UI navigates back to the
+ // profile list after a successful save.
+ await waitForTestId(page, "add-llm-profile");
+ });
+
+ // ── Verify: the proxy base_url survived the Basic-tab save ──
+ await test.step("verify base_url is preserved after save", async () => {
+ const config = await getProfileConfig(request, PROXY_PROFILE);
+ expect(
+ config.base_url,
+ "base_url must be the All-Hands proxy URL after a Basic-tab re-save; " +
+ "dropping it strands the profile (issue #1146)",
+ ).toBe(OPENHANDS_PROXY_BASE_URL);
+ expect(config.model).toBe(LITELLM_PROXY_MODEL);
+ });
+
+ // ── Verify: the profile also looks correct after a page reload ──
+ await test.step("profile survives page reload", async () => {
+ await page.reload({ waitUntil: "domcontentloaded" });
+ await waitForTestId(page, "add-llm-profile");
+
+ // Re-read via API to confirm persistence is durable
+ const config = await getProfileConfig(request, PROXY_PROFILE);
+ expect(config.base_url).toBe(OPENHANDS_PROXY_BASE_URL);
+ expect(config.model).toBe(LITELLM_PROXY_MODEL);
+ });
+ });
+});