Use Claude Code, Codex, VSCode, Pi, and OpenCode (and 6 other harnesses) for free (1.3B+ free tokens) from your terminal, app, IDE, or phone, and now from the browser with native browser sessions (multi-harness + multi-model) like OpenClaw (voice supported + ToS friendly)
Python
"""Rendered provider-setup regressions for the local Admin UI."""
import pytest
from playwright.sync_api import ConsoleMessage, Page, Route, ViewportSize, expect
from e2e.provider_support import api_key_input, close_provider, open_provider
@pytest.mark.parametrize(
"admin_base_url", [{"MODEL": "github_models/openai/old"}], indirect=True
)
def test_retired_provider_is_absent_and_default_setup_remains_available(
page: Page, admin_base_url: str
):
_open_admin(page, admin_base_url, {"width": 1280, "height": 720})
expect(page.locator('[data-provider="github_models"]')).to_have_count(0)
expect(page.locator("#field-GITHUB_MODELS_TOKEN")).to_have_count(0)
card = page.locator('[data-provider="nvidia_nim"]')
expect(card.get_by_role("button", name="Configure", exact=True)).to_have_class(
"primary-button"
)
card.locator("[data-provider-settings]").click()
expect(page.get_by_role("button", name="Add key", exact=True)).to_be_focused()
close_provider(page)
page.get_by_role("button", name="Model Config", exact=True).click()
expect(page.locator("#field-MODEL")).to_have_value(
"nvidia_nim/nvidia/nemotron-3-ultra-550b-a55b"
)
def _open_admin(
page: Page,
admin_base_url: str,
viewport: ViewportSize,
) -> None:
page.set_viewport_size(viewport)
page.emulate_media(reduced_motion="reduce")
page.goto(f"{admin_base_url}/admin")
expect(page.locator("#messageArea")).to_have_text("")
@pytest.mark.parametrize(
("viewport", "desktop"),
(
({"width": 1280, "height": 720}, True),
({"width": 390, "height": 844}, False),
),
)
def test_missing_provider_configuration_opens_modal_and_focuses_exact_field(
page: Page,
admin_base_url: str,
viewport: ViewportSize,
desktop: bool,
) -> None:
_open_admin(page, admin_base_url, viewport)
card = page.locator('[data-provider="nvidia_nim"]')
key_input = page.get_by_role("button", name="Add key", exact=True)
expect(card.get_by_role("button", name="Configure", exact=True)).to_have_class(
"primary-button"
)
expect(card.locator("[data-provider-settings]")).to_have_attribute(
"aria-haspopup", "dialog"
)
expect(card.get_by_role("button", name="Refresh models", exact=True)).to_have_count(
0
)
expect(key_input).to_have_count(0)
card.locator("[data-provider-settings]").click()
expect(key_input).to_be_in_viewport()
expect(key_input).to_be_focused()
if desktop:
sidebar = page.locator(".sidebar")
expect(sidebar).to_have_css("position", "sticky")
assert (
round(
float(
sidebar.evaluate("element => element.getBoundingClientRect().top")
)
)
== 0
)
def test_desktop_sidebar_stays_pinned_at_document_bottom(
page: Page,
admin_base_url: str,
) -> None:
_open_admin(page, admin_base_url, {"width": 1280, "height": 720})
sidebar = page.locator(".sidebar")
page.evaluate("window.scrollTo(0, document.documentElement.scrollHeight)")
sidebar_top = float(
sidebar.evaluate("element => element.getBoundingClientRect().top")
)
assert sidebar_top == pytest.approx(0, abs=0.5)
def test_configured_provider_check_keeps_readiness_and_adds_models(
page: Page,
admin_base_url: str,
) -> None:
_open_admin(page, admin_base_url, {"width": 1280, "height": 720})
card = page.locator('[data-provider="open_router"]')
expect(card.locator(".provider-check-result")).to_have_text("3 models available")
expect(card.get_by_role("button", name="Edit", exact=True)).to_have_class(
"secondary-button"
)
expect(card.locator("[data-provider-settings]")).to_have_attribute(
"aria-haspopup", "dialog"
)
card.locator("[data-provider-settings]").click()
page.locator("#providerDialog").get_by_role(
"button", name="Refresh models", exact=True
).click()
expect(card.locator(".provider-check-result")).to_have_text("3 models available")
expect(card.get_by_role("button", name="Edit", exact=True)).to_have_class(
"secondary-button"
)
close_provider(page)
page.get_by_role("button", name="Model Config", exact=True).click()
fable = page.get_by_role(
"combobox",
name="Fable Override default",
exact=True,
)
page.get_by_role("button", name="Show Fable Override options", exact=True).click()
expect(page.get_by_role("listbox").get_by_role("option")).to_have_count(1)
expect(page.get_by_role("option", name="None", exact=True)).to_be_visible()
fable.fill("vendor/model-a")
expect(
page.get_by_role("option", name="open_router/vendor/model-a", exact=True)
).to_be_visible()
@pytest.mark.parametrize("models", [[], ["only-model"]])
def test_startup_model_count_handles_empty_and_single_model_catalogs(
page: Page, admin_base_url: str, models: list[str]
) -> None:
def status(route: Route) -> None:
data = route.fetch().json()
data["startup"]["providers"]["open_router"] = "ready"
data["cached_models"]["open_router"] = models
route.fulfill(json=data)
page.route("**/admin/api/status", status)
_open_admin(page, admin_base_url, {"width": 1280, "height": 720})
expect(page.locator('[data-provider-check-result="open_router"]')).to_have_text(
"1 model available" if models else "0 models available"
)
expect(page.locator('[data-provider-check-result="nvidia_nim"]')).to_be_hidden()
@pytest.mark.parametrize("manual_result", ["pending", "success", "failure"])
def test_delayed_startup_status_does_not_replace_a_manual_provider_check(
page: Page, admin_base_url: str, manual_result: str
) -> None:
_open_admin(page, admin_base_url, {"width": 1280, "height": 720})
page.wait_for_function("!state.startupRequest && !state.startupTimer")
snapshot = page.request.get(f"{admin_base_url}/admin/api/status").json()
startup: list[Route] = []
manual: list[Route] = []
def hold_first_status(route: Route) -> None:
# Only the stale snapshot is held. Later refreshes must finish normally.
if startup:
route.continue_()
else:
startup.append(route)
page.evaluate("window.startupRequests = (window.startupRequests || 0) + 1")
def hold_manual(route: Route) -> None:
manual.append(route)
page.evaluate("window.manualRequests = (window.manualRequests || 0) + 1")
page.route("**/admin/api/status", hold_first_status)
page.route("**/admin/api/providers/open_router/test", hold_manual)
page.evaluate("void refreshStartup()")
page.wait_for_function("window.startupRequests >= 1")
dialog = open_provider(page, "open_router")
dialog.get_by_role("button", name="Refresh models", exact=True).click()
page.wait_for_function("window.manualRequests >= 1")
expected = "Checking..."
if manual_result != "pending":
manual.pop().fulfill(
json={
"ok": manual_result == "success",
"models": ["one", "two"],
"message": "Could not refresh this provider's models.",
}
)
expected = (
"2 models available"
if manual_result == "success"
else "Unavailable: Could not refresh this provider's models."
)
result = page.locator('[data-provider-check-result="open_router"]')
expect(result).to_have_text(expected)
with page.expect_response("**/admin/api/status") as response:
startup[0].fulfill(json=snapshot)
response.value.finished()
page.wait_for_function("!state.startupRequest")
expect(result).to_have_text(expected)
expect(dialog.locator("#providerDialogCheck")).to_have_text(expected)
if manual_result == "pending":
manual.pop().fulfill(json={"ok": True, "models": ["one", "two"]})
expect(result).to_have_text("2 models available")
@pytest.mark.parametrize("availability_first", [False, True])
@pytest.mark.parametrize("discovery", ["starting", "ready", "failed"])
@pytest.mark.parametrize("available", [False, True])
def test_local_model_discovery_takes_precedence_over_reachability(
page: Page,
admin_base_url: str,
availability_first: bool,
discovery: str,
available: bool,
) -> None:
startup: list[Route] = []
availability: list[Route] = []
def status(route: Route) -> None:
if startup:
route.fulfill(json=snapshot)
else:
startup.append(route)
page.route("**/admin/api/status", status)
page.route(
"**/admin/api/providers/lmstudio/local-status",
lambda route: availability.append(route),
)
_open_admin(page, admin_base_url, {"width": 1280, "height": 720})
page.wait_for_function("!!state.startupRequest && !!state.localStatusRequest")
snapshot = page.request.get(f"{admin_base_url}/admin/api/status").json()
snapshot["startup"]["providers"]["lmstudio"] = discovery
snapshot["cached_models"]["lmstudio"] = ["local-model"]
primary = {
"starting": "Checking models…",
"ready": "1 model available",
"failed": "Could not load models. Check the provider's settings and retry.",
}[discovery]
probe = {
"status": "reachable" if available else "offline",
"base_url": "http://localhost:1234/v1",
"status_code": 200 if available else 503,
"probe_path": "/models",
"latency_ms": 12.5,
}
diagnostic = (
"Reachable: http://localhost:1234/v1"
if available
else "Unavailable: http://localhost:1234/v1 returned HTTP 503"
) + " (GET /models, 12.5 ms)"
result = page.locator('[data-provider-check-result="lmstudio"]')
detail = page.locator('[data-provider-availability-result="lmstudio"]')
dialog = open_provider(page, "lmstudio")
if availability_first:
availability.pop().fulfill(json=probe)
expect(result).to_have_text(diagnostic)
expect(detail).to_be_hidden()
startup[0].fulfill(json=snapshot)
expect(result).to_have_text(primary)
if not availability_first:
availability.pop().fulfill(json=probe)
page.wait_for_function("!state.localStatusRequest")
expect(result).to_have_text(primary)
expect(detail).to_have_text(diagnostic)
expect(detail).to_have_class(
"provider-availability-result " + ("ok" if available else "error")
)
expect(dialog.locator("#providerDialogCheck")).to_have_text(primary)
expect(dialog.locator("#providerDialogAvailability")).to_have_text(diagnostic)
if discovery == "starting":
with page.expect_response("**/admin/api/status"):
page.evaluate("void refreshStartup()")
def test_provider_check_failure_is_separate_and_never_exposes_exception_text(
page: Page,
admin_base_url: str,
) -> None:
console_messages: list[str] = []
def record_console(message: ConsoleMessage) -> None:
console_messages.append(message.text)
page.on("console", record_console)
_open_admin(page, admin_base_url, {"width": 1280, "height": 720})
card = page.locator('[data-provider="groq"]')
expect(card.locator(".provider-check-result")).to_have_text(
"Could not load models. Check the provider's settings and retry."
)
card.locator("[data-provider-settings]").click()
page.locator("#providerDialog").get_by_role(
"button", name="Refresh models", exact=True
).click()
result = card.locator(".provider-check-result")
expect(result).to_have_text(
"Unavailable: Could not load models. Check the provider's settings and retry."
)
expect(card.get_by_role("button", name="Edit", exact=True)).to_have_class(
"secondary-button"
)
page_text = page.locator("body").inner_text()
secret = "CREDENTIAL[unrecognized-format-987654321]"
assert secret not in page_text
assert "RuntimeError" not in page_text
assert secret not in "\n".join(console_messages)
def test_multi_field_provider_targets_first_missing_configuration(
page: Page,
admin_base_url: str,
) -> None:
_open_admin(page, admin_base_url, {"width": 1280, "height": 720})
card = page.locator('[data-provider="cloudflare"]')
account_input = page.locator("#field-CLOUDFLARE_ACCOUNT_ID")
expect(card.get_by_role("button", name="Configure", exact=True)).to_have_class(
"primary-button"
)
expect(account_input).to_have_count(0)
card.locator("[data-provider-settings]").click()
expect(account_input).to_be_in_viewport()
expect(account_input).to_be_focused()
def test_admin_loading_finishes_before_local_availability_checks(
page: Page, admin_base_url: str
) -> None:
pending: list[Route] = []
def hold(route):
pending.append(route)
page.evaluate("window.localChecks = (window.localChecks || 0) + 1")
page.route("**/admin/api/providers/*/local-status", hold)
_open_admin(page, admin_base_url, {"width": 1280, "height": 720})
open_provider(page, "nvidia_nim")
key = api_key_input(page, "NVIDIA_NIM_API_KEY")
key.fill("unsaved-key")
expect(page.locator("#dirtyState")).to_have_text("No changes")
expect(page.locator("#saveProvider")).to_be_enabled()
page.wait_for_function("window.localChecks === 3")
for route in pending:
payload = route.fetch().json()
if payload["provider_id"] == "llamacpp":
payload.update(status="offline", label="Offline", status_code=503)
elif payload["provider_id"] == "ollama":
payload.update(status="missing_url", label="Missing URL", base_url="")
route.fulfill(json=payload)
expect(page.locator('[data-provider-availability-result="lmstudio"]')).to_have_text(
"Reachable: http://localhost:1234/v1"
)
expect(page.locator('[data-provider-availability-result="llamacpp"]')).to_have_text(
"Unavailable: http://localhost:8080/v1 returned HTTP 503"
)
expect(page.locator('[data-provider-availability-result="ollama"]')).to_be_hidden()
expect(
page.locator('[data-provider="lmstudio"]').get_by_role(
"button", name="Edit", exact=True
)
).to_have_class("secondary-button")
expect(key).to_have_value("unsaved-key")
expect(page.locator("#dirtyState")).to_have_text("No changes")
expect(page.locator("#messageArea")).to_have_text("")
@pytest.mark.parametrize("failure", ["http", "network"])
def test_local_availability_failure_does_not_fail_admin_loading(
page: Page, admin_base_url: str, failure: str
) -> None:
pending: list[Route] = []
errors: list[str] = []
page.on("pageerror", lambda error: errors.append(str(error)))
page.route(
"**/admin/api/providers/lmstudio/local-status",
lambda route: pending.append(route),
)
with page.expect_request("**/admin/api/providers/lmstudio/local-status"):
page.goto(f"{admin_base_url}/admin")
open_provider(page, "nvidia_nim")
expect(api_key_input(page, "NVIDIA_NIM_API_KEY")).to_be_editable()
close_provider(page)
if failure == "http":
pending.pop().fulfill(status=503, json={"detail": "private-diagnostic-marker"})
else:
pending.pop().abort()
for provider_id in ("lmstudio", "llamacpp", "ollama"):
card = page.locator(f'[data-provider="{provider_id}"]')
if provider_id == "lmstudio":
expect(card.locator(".provider-availability-result")).to_have_text(
"Availability check failed. Use Test to retry."
)
else:
expect(card.locator(".provider-availability-result")).to_contain_text(
"Reachable:"
)
expect(card.get_by_role("button", name="Edit", exact=True)).to_have_class(
"secondary-button"
)
dialog = open_provider(page, provider_id)
expect(dialog.get_by_role("button", name="Test", exact=True)).to_be_enabled()
close_provider(page)
expect(page.locator('[data-provider-check-result="open_router"]')).to_have_text(
"3 models available"
)
expect(page.locator("#messageArea")).to_have_text("")
assert "private-diagnostic-marker" not in page.locator("body").inner_text()
assert errors == []
@pytest.mark.parametrize(
"viewport",
({"width": 1280, "height": 720}, {"width": 390, "height": 844}),
)
def test_local_provider_diagnostics_are_visible_on_cards_and_in_the_dialog(
page: Page, admin_base_url: str, viewport: ViewportSize
) -> None:
errors: list[str] = []
page.on("pageerror", lambda error: errors.append(str(error)))
timeout_message = (
"The local provider availability check timed out. "
"Check that its server is responsive and retry."
)
results = {
"lmstudio": {
"provider_id": "lmstudio",
"status": "reachable",
"base_url": "http://localhost:1234/v1",
"probe_path": "/models",
"latency_ms": 12.5,
"error_type": None,
},
"llamacpp": {
"provider_id": "llamacpp",
"status": "offline",
"base_url": "http://localhost:8080/v1",
"status_code": 503,
"probe_path": "/models",
"latency_ms": 8,
"error_type": "http_error",
},
"ollama": {
"provider_id": "ollama",
"status": "offline",
"base_url": "http://localhost:11434",
"message": timeout_message,
"probe_path": "/api/tags",
"latency_ms": 1500,
"error_type": "timeout",
"private_diagnostic": "private-credential-marker",
},
}
def reply(route: Route) -> None:
provider_id = route.request.url.split("/")[-2]
route.fulfill(json=results[provider_id])
page.route("**/admin/api/providers/*/local-status", reply)
_open_admin(page, admin_base_url, viewport)
expect(page.locator('[data-provider-availability-result="lmstudio"]')).to_have_text(
"Reachable: http://localhost:1234/v1 (GET /models, 12.5 ms)"
)
expect(page.locator('[data-provider-availability-result="llamacpp"]')).to_have_text(
"Unavailable: http://localhost:8080/v1 returned HTTP 503 (GET /models, 8 ms)"
)
expected_timeout = f"Unavailable: {timeout_message} (GET /api/tags, 1500 ms)"
expect(page.locator('[data-provider-availability-result="ollama"]')).to_have_text(
expected_timeout
)
dialog = open_provider(page, "ollama")
expect(dialog.locator("#providerDialogAvailability")).to_have_text(expected_timeout)
expect(dialog.locator("#providerDialogAvailability")).to_be_in_viewport()
expect(dialog.get_by_role("button", name="Test", exact=True)).to_be_enabled()
assert "private-credential-marker" not in page.locator("body").inner_text()
assert errors == []
@pytest.mark.parametrize("manual_finished", [False, True])
def test_manual_provider_test_takes_precedence_over_automatic_availability(
page: Page, admin_base_url: str, manual_finished: bool
) -> None:
availability: list[Route] = []
manual: list[Route] = []
page.route(
"**/admin/api/providers/lmstudio/local-status",
lambda route: availability.append(route),
)
page.route(
"**/admin/api/providers/lmstudio/test", lambda route: manual.append(route)
)
_open_admin(page, admin_base_url, {"width": 1280, "height": 720})
card = page.locator('[data-provider="lmstudio"]')
dialog = open_provider(page, "lmstudio")
with page.expect_request("**/admin/api/providers/lmstudio/test"):
dialog.get_by_role("button", name="Test", exact=True).click()
result = card.locator(".provider-check-result")
expect(result).to_have_text("Checking...")
if manual_finished:
manual.pop().fulfill(
json={
"provider_id": "lmstudio",
"ok": False,
"message": "Could not refresh this provider's models.",
}
)
expect(result).to_have_text(
"Unavailable: Could not refresh this provider's models."
)
with page.expect_response(
"**/admin/api/providers/lmstudio/local-status"
) as response:
if manual_finished:
availability.pop().fulfill(status=503, json={"detail": "Check failed"})
else:
availability.pop().continue_()
response.value.finished()
page.evaluate("() => new Promise(requestAnimationFrame)")
other = page.locator('[data-provider-availability-result="ollama"]')
if manual_finished:
expect(result).to_have_text(
"Unavailable: Could not refresh this provider's models."
)
expect(other).to_have_text("Reachable: http://localhost:11434")
else:
expect(result).to_have_text("Checking...")
expect(other).to_have_text("Reachable: http://localhost:11434")
manual.pop().fulfill(
json={"provider_id": "lmstudio", "ok": True, "models": ["local-model"]}
)
expect(result).to_have_text("1 model available")
expect(dialog.get_by_role("button", name="Test", exact=True)).to_be_enabled()
def test_xkiro_uses_standard_provider_configuration(page: Page, admin_base_url: str):
_open_admin(page, admin_base_url, {"width": 1280, "height": 720})
card = page.locator('[data-provider="xkiro"]')
expect(card.get_by_role("button", name="Configure", exact=True)).to_be_visible()
expect(card.locator('a[href="https://xkiro.com/"]')).to_be_visible()
dialog = open_provider(page, "xkiro")
expect(dialog.get_by_role("button", name="Add key", exact=True)).to_be_focused()
expect(api_key_input(page, "XKIRO_API_KEY")).to_have_attribute("type", "password")
expect(dialog.locator("#field-XKIRO_PROXY")).to_be_visible()
expect(dialog).to_contain_text("xkiro.com/dashboard/api/keys")
close_provider(page)
def test_anthropic_standard_modal_groups_key_workspace_and_proxy(
page: Page, admin_base_url: str
):
submissions = []
def save(route: Route):
submissions.append(route.request.post_data_json)
route.fulfill(
json={
"applied": True,
"credential_checks": [],
"restart": {"required": False},
}
)
page.route("**/admin/api/config/apply", save)
_open_admin(page, admin_base_url, {"width": 1280, "height": 720})
card = page.locator('[data-provider="anthropic"]')
expect(card.get_by_role("button", name="Configure", exact=True)).to_be_visible()
dialog = open_provider(page, "anthropic")
key = api_key_input(page, "ANTHROPIC_API_KEY")
expect(key).to_have_attribute("type", "password")
expect(key).to_be_focused()
expect(dialog.locator("#field-ANTHROPIC_WORKSPACE_ID")).to_be_visible()
expect(dialog.locator("#field-ANTHROPIC_PROXY")).to_be_visible()
key.fill("test-api-key")
dialog.locator("#field-ANTHROPIC_WORKSPACE_ID").fill("wrkspc_test")
dialog.get_by_role("button", name="Save", exact=True).click()
expect(dialog).not_to_be_visible()
assert len(submissions) == 1
assert set(submissions[0]["values"]) == {
"ANTHROPIC_API_KEY",
"ANTHROPIC_WORKSPACE_ID",
}
assert submissions[0]["values"]["ANTHROPIC_API_KEY"]["entries"] == [
{"value": "test-api-key"}
]
assert submissions[0]["values"]["ANTHROPIC_WORKSPACE_ID"] == "wrkspc_test"