Skip to content

πŸ™Œ OpenHands: AI-Driven Development

fix: profiles as source of truth + keep one active (local mode) (#1128)

Vasco Schiavo5d50a3aafb81
diff --git a/.github/workflows/mock-llm-docker-e2e.yml b/.github/workflows/mock-llm-docker-e2e.yml
index 00a991694b..2479dcffe7 100644
--- a/.github/workflows/mock-llm-docker-e2e.yml
+++ b/.github/workflows/mock-llm-docker-e2e.yml
@@ -260,8 +260,11 @@ jobs:
npm run test:e2e:mock-llm:docker &
PW_PID=$!
- # Wait up to 5 min for tests to complete.
- deadline=$((SECONDS + 300))
+ # Wait for tests to complete. Playwright's globalTimeout is 600s
+ # (10 min) in CI; we add 60s buffer for container startup/teardown.
+ # .results.json is flushed after every single test, so even if we
+ # hit the deadline mid-suite the report script still has data.
+ deadline=$((SECONDS + 660))
while [ "$SECONDS" -lt "$deadline" ]; do
if ! kill -0 "$PW_PID" 2>/dev/null; then
break
@@ -273,12 +276,16 @@ jobs:
sleep 2
done
- # If Playwright is still running (teardown hang), give it 5s
- # grace then force-kill.
+ # If Playwright is still running (teardown hang or deadline hit),
+ # give it 5s grace then force-kill.
if kill -0 "$PW_PID" 2>/dev/null; then
sleep 5
if kill -0 "$PW_PID" 2>/dev/null; then
- echo "::warning::Killing lingering Playwright process (teardown hung)"
+ if [ -f "$DONE_MARKER" ]; then
+ echo "::warning::Killing lingering Playwright process (teardown hung)"
+ else
+ echo "::warning::Killing Playwright process (deadline reached, tests still running)"
+ fi
kill "$PW_PID" 2>/dev/null
sleep 5
kill -9 "$PW_PID" 2>/dev/null
diff --git a/.github/workflows/mock-llm-e2e.yml b/.github/workflows/mock-llm-e2e.yml
index d4bba03a84..2361b7ce93 100644
--- a/.github/workflows/mock-llm-e2e.yml
+++ b/.github/workflows/mock-llm-e2e.yml
@@ -127,11 +127,13 @@ jobs:
npm run test:e2e:mock-llm &
PW_PID=$!
- # Wait up to 5 min for tests to complete. The custom reporter
- # writes .tests-done (content: "passed" or "failed") in onEnd()
- # β€” which fires BEFORE webServer teardown starts. This lets us
- # detect completion immediately on both pass AND fail.
- deadline=$((SECONDS + 300))
+ # Wait for tests to complete. Playwright's globalTimeout is 600s
+ # (10 min) in CI; we add 60s buffer for webServer startup/teardown.
+ # The custom DoneMarkerReporter writes .tests-done only after ALL
+ # tests finish (pass or fail), before webServer teardown begins.
+ # .results.json is flushed after every single test, so even if we
+ # hit the deadline mid-suite the report script still has data.
+ deadline=$((SECONDS + 660))
while [ "$SECONDS" -lt "$deadline" ]; do
if ! kill -0 "$PW_PID" 2>/dev/null; then
break
@@ -143,15 +145,16 @@ jobs:
sleep 2
done
- # If Playwright is still running (teardown hang), give it 5s
- # grace then force-kill. The hang is expected: Playwright's
- # webServer teardown waits for the multi-process agent-server
- # tree (node β†’ uvx β†’ Python β†’ uvicorn workers) to fully exit,
- # which takes longer than our marker-based detection.
+ # If Playwright is still running (teardown hang or deadline hit),
+ # give it 5s grace then force-kill.
if kill -0 "$PW_PID" 2>/dev/null; then
sleep 5
if kill -0 "$PW_PID" 2>/dev/null; then
- echo "::warning::Killing lingering Playwright process (teardown hung)"
+ if [ -f "$DONE_MARKER" ]; then
+ echo "::warning::Killing lingering Playwright process (teardown hung)"
+ else
+ echo "::warning::Killing Playwright process (deadline reached, tests still running)"
+ fi
kill "$PW_PID" 2>/dev/null
sleep 5
kill -9 "$PW_PID" 2>/dev/null
diff --git a/__tests__/components/chat/chat-interface.test.tsx b/__tests__/components/chat/chat-interface.test.tsx
index b9f6d51d03..5c09cc6bb5 100644
--- a/__tests__/components/chat/chat-interface.test.tsx
+++ b/__tests__/components/chat/chat-interface.test.tsx
@@ -43,6 +43,11 @@ vi.mock("#/hooks/use-send-message", () => ({
vi.mock("#/hooks/query/use-config");
vi.mock("#/hooks/mutation/use-unified-upload-files");
+// Treat the LLM as configured by default so the "not configured" gate/banner
+// stays inert for these tests (its own behavior is covered elsewhere).
+vi.mock("#/hooks/use-llm-configured", () => ({
+ useLlmConfigured: () => ({ isConfigured: true, isLoading: false }),
+}));
vi.mock("#/hooks/use-conversation-id", () => ({
useConversationId: vi.fn(),
useOptionalConversationId: vi.fn(),
diff --git a/__tests__/components/features/home/home-chat-launcher.test.tsx b/__tests__/components/features/home/home-chat-launcher.test.tsx
index 63c3e0f210..b86f6b2b04 100644
--- a/__tests__/components/features/home/home-chat-launcher.test.tsx
+++ b/__tests__/components/features/home/home-chat-launcher.test.tsx
@@ -14,6 +14,7 @@ const sendMessageWithAttachments = vi.fn();
const mockClearAllFiles = vi.fn();
const enqueueHomeTaskPendingMessage = vi.fn();
const mockDisplayErrorToast = vi.fn();
+const mockUseLlmConfigured = vi.fn();
let mockImages: File[] = [];
let mockFiles: File[] = [];
@@ -63,6 +64,10 @@ vi.mock("#/contexts/active-backend-context", () => ({
useActiveBackend: () => mockUseActiveBackend(),
}));
+vi.mock("#/hooks/use-llm-configured", () => ({
+ useLlmConfigured: () => mockUseLlmConfigured(),
+}));
+
vi.mock("#/hooks/use-is-creating-conversation", () => ({
useIsCreatingConversation: () => false,
}));
@@ -238,6 +243,10 @@ describe("HomeChatLauncher", () => {
mockImages = [];
mockFiles = [];
mockUseActiveBackend.mockReturnValue(localBackend);
+ mockUseLlmConfigured.mockReturnValue({
+ isConfigured: true,
+ isLoading: false,
+ });
enqueueHomeTaskPendingMessage.mockResolvedValue(undefined);
sendMessageWithAttachments.mockResolvedValue({
text: "hello world",
@@ -281,6 +290,22 @@ describe("HomeChatLauncher", () => {
);
});
+ it("disables the chat input and won't create a conversation when no LLM is configured", async () => {
+ mockUseLlmConfigured.mockReturnValue({
+ isConfigured: false,
+ isLoading: false,
+ });
+ const createSpy = vi
+ .spyOn(AgentServerConversationService, "createConversation")
+ .mockResolvedValue(makeConversationResponse());
+
+ renderLauncher();
+
+ // The send is disabled, so the agent can't be handed a request it can't run.
+ expect(screen.getByTestId("stub-chat-submit")).toBeDisabled();
+ expect(createSpy).not.toHaveBeenCalled();
+ });
+
it("passes the picked workspace path as working_dir on a local backend", async () => {
const createSpy = vi
.spyOn(AgentServerConversationService, "createConversation")
diff --git a/__tests__/components/home/llm-not-configured-banner.test.tsx b/__tests__/components/home/llm-not-configured-banner.test.tsx
index 975b938477..6258b03ad5 100644
--- a/__tests__/components/home/llm-not-configured-banner.test.tsx
+++ b/__tests__/components/home/llm-not-configured-banner.test.tsx
@@ -104,20 +104,21 @@ describe("LlmNotConfiguredBanner", () => {
).toBeInTheDocument();
});
- it("stays hidden once an LLM API key is configured", async () => {
- // Arrange
+ it("warns when only a stale settings key is set but no active profile (local)", async () => {
+ // Arrange: the delete-all state β€” agent_settings still holds a key, but no
+ // profile backs it. In local mode profiles are the source of truth, so a
+ // lingering settings key must not count as configured.
vi.spyOn(SettingsService, "getSettings").mockResolvedValue(
buildSettings({ llm_api_key_set: true }),
);
// Act
renderBanner();
- await screen.findByTestId("queries-settled");
// Assert
expect(
- screen.queryByTestId("home-llm-not-configured-banner"),
- ).not.toBeInTheDocument();
+ await screen.findByTestId("home-llm-not-configured-banner"),
+ ).toBeInTheDocument();
});
it("stays hidden once an active LLM profile has a saved API key", async () => {
diff --git a/__tests__/components/settings/llm-profiles/profile-actions-menu.test.tsx b/__tests__/components/settings/llm-profiles/profile-actions-menu.test.tsx
index 21bc151fd9..cd4c0c09bf 100644
--- a/__tests__/components/settings/llm-profiles/profile-actions-menu.test.tsx
+++ b/__tests__/components/settings/llm-profiles/profile-actions-menu.test.tsx
@@ -11,7 +11,6 @@ vi.mock("react-i18next", () => ({
"BUTTON$RENAME": "Rename",
"SETTINGS$PROFILE_SET_ACTIVE": "Set as active",
"BUTTON$DELETE": "Delete",
- "SETTINGS$PROFILE_CANNOT_DELETE_ACTIVE": "Cannot delete the active profile",
};
return translations[key] || key;
},
@@ -109,10 +108,12 @@ describe("ProfileActionsMenu", () => {
expect(setActiveButton).toBeDisabled();
});
- it("disables Delete button when isActive is true", () => {
+ it("keeps Delete enabled even when isActive is true", () => {
+ // The active profile is deletable; useEnsureActiveProfile promotes another
+ // profile to active afterwards so one is always active in local mode.
render(<ProfileActionsMenu {...defaultProps} isActive />);
- expect(screen.getByTestId("profile-delete")).toBeDisabled();
+ expect(screen.getByTestId("profile-delete")).not.toBeDisabled();
});
it("enables Delete button when isActive is false", () => {
diff --git a/__tests__/hooks/use-ensure-active-profile.test.tsx b/__tests__/hooks/use-ensure-active-profile.test.tsx
new file mode 100644
index 0000000000..4946ad1593
--- /dev/null
+++ b/__tests__/hooks/use-ensure-active-profile.test.tsx
@@ -0,0 +1,86 @@
+import { renderHook } from "@testing-library/react";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+import { useEnsureActiveProfile } from "#/hooks/use-ensure-active-profile";
+
+const mockActivate = vi.fn();
+const mockUseActiveBackend = vi.fn();
+const mockUseLlmProfiles = vi.fn();
+
+vi.mock("#/contexts/active-backend-context", () => ({
+ useActiveBackend: () => mockUseActiveBackend(),
+}));
+vi.mock("#/hooks/query/use-llm-profiles", () => ({
+ useLlmProfiles: () => mockUseLlmProfiles(),
+}));
+vi.mock("#/hooks/mutation/use-activate-llm-profile", () => ({
+ useActivateLlmProfile: () => ({ mutate: mockActivate, isPending: false }),
+}));
+
+const local = { backend: { id: "b1", kind: "local" }, orgId: null };
+const cloud = { backend: { id: "c1", kind: "cloud" }, orgId: null };
+const profile = (name: string, api_key_set = true) => ({
+ name,
+ model: "m",
+ base_url: null,
+ api_key_set,
+});
+
+describe("useEnsureActiveProfile", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ mockUseActiveBackend.mockReturnValue(local);
+ });
+
+ it("activates the first profile when none is active", () => {
+ mockUseLlmProfiles.mockReturnValue({
+ data: { profiles: [profile("a"), profile("b")], active_profile: null },
+ });
+ renderHook(() => useEnsureActiveProfile());
+ expect(mockActivate).toHaveBeenCalledWith("a");
+ });
+
+ it("activates a remaining profile when the active one is dangling", () => {
+ mockUseLlmProfiles.mockReturnValue({
+ data: { profiles: [profile("b")], active_profile: "deleted" },
+ });
+ renderHook(() => useEnsureActiveProfile());
+ expect(mockActivate).toHaveBeenCalledWith("b");
+ });
+
+ it("prefers a profile that has a key over an earlier keyless one", () => {
+ mockUseLlmProfiles.mockReturnValue({
+ data: {
+ profiles: [profile("nokey", false), profile("withkey")],
+ active_profile: null,
+ },
+ });
+ renderHook(() => useEnsureActiveProfile());
+ expect(mockActivate).toHaveBeenCalledWith("withkey");
+ });
+
+ it("does nothing when the active profile is valid", () => {
+ mockUseLlmProfiles.mockReturnValue({
+ data: { profiles: [profile("a")], active_profile: "a" },
+ });
+ renderHook(() => useEnsureActiveProfile());
+ expect(mockActivate).not.toHaveBeenCalled();
+ });
+
+ it("does nothing when there are no profiles", () => {
+ mockUseLlmProfiles.mockReturnValue({
+ data: { profiles: [], active_profile: "dangling" },
+ });
+ renderHook(() => useEnsureActiveProfile());
+ expect(mockActivate).not.toHaveBeenCalled();
+ });
+
+ it("does nothing on a cloud backend", () => {
+ mockUseActiveBackend.mockReturnValue(cloud);
+ mockUseLlmProfiles.mockReturnValue({
+ data: { profiles: [profile("a")], active_profile: null },
+ });
+ renderHook(() => useEnsureActiveProfile());
+ expect(mockActivate).not.toHaveBeenCalled();
+ });
+});
diff --git a/src/components/features/chat/chat-interface.tsx b/src/components/features/chat/chat-interface.tsx
index 1e4b28e945..7f2ab7a03b 100644
--- a/src/components/features/chat/chat-interface.tsx
+++ b/src/components/features/chat/chat-interface.tsx
@@ -27,6 +27,8 @@ import { useErrorMessageStore } from "#/stores/error-message-store";
import { useOptimisticUserMessageStore } from "#/stores/optimistic-user-message-store";
import { SERVER_CONNECTION_ERROR_MESSAGE } from "#/constants/server-connection-error";
import { ErrorMessageBanner } from "./error-message-banner";
+import { LlmNotConfiguredBanner } from "#/components/features/home/llm-not-configured-banner";
+import { useLlmConfigured } from "#/hooks/use-llm-configured";
import { Messages } from "#/components/conversation-events/chat/messages";
import { PendingUserMessages } from "./pending-user-messages";
import { useUnifiedUploadFiles } from "#/hooks/mutation/use-unified-upload-files";
@@ -106,6 +108,12 @@ export function ChatInterface() {
const isArchivedConversation =
sandboxStatus === "MISSING" || sandboxStatus === "ERROR";
+ // Block sending in a resumed conversation that has no usable LLM, and show
+ // the same setup banner as the home screen so the dead end is explained.
+ const { isConfigured: isLlmConfigured, isLoading: isLlmConfigLoading } =
+ useLlmConfigured();
+ const llmBlocked = !isLlmConfigLoading && !isLlmConfigured;
+
// Disable Build button while agent is running (streaming)
const isAgentRunning =
curAgentState === AgentState.RUNNING ||
@@ -450,7 +458,12 @@ export function ChatInterface() {
!isChatLoading &&
!isProvisioningTask &&
totalEvents === 0 &&
- !isArchivedConversation && (
+ !isArchivedConversation &&
+ // With no usable LLM the suggestions can't be acted on (the input is
+ // disabled). They're also a `pointer-events-auto` overlay that would
+ // sit over the LlmNotConfiguredBanner below and swallow clicks on its
+ // setup button β€” so hide them and let the banner be the lone CTA.
+ !llmBlocked && (
<ChatSuggestions
onSuggestionsClick={(message) => setMessageToSend(message)}
/>
@@ -534,6 +547,8 @@ export function ChatInterface() {
/>
)}
+ {llmBlocked && !isArchivedConversation && <LlmNotConfiguredBanner />}
+
{isArchivedConversation ? (
// Archived / sandbox-error: show a read-only notice in place of
// the chat input. The conversation history above is still visible.
@@ -582,7 +597,7 @@ export function ChatInterface() {
<InteractiveChatBox
onSubmit={handleSendMessage}
- disabled={isNewConversationPending}
+ disabled={isNewConversationPending || llmBlocked}
/>
</div>
)}
diff --git a/src/components/features/home/home-chat-launcher.tsx b/src/components/features/home/home-chat-launcher.tsx
index 2473decc9f..5e7bd432cb 100644
--- a/src/components/features/home/home-chat-launcher.tsx
+++ b/src/components/features/home/home-chat-launcher.tsx
@@ -6,6 +6,7 @@ import { useActiveBackend } from "#/contexts/active-backend-context";
import { useCreateConversation } from "#/hooks/mutation/use-create-conversation";
import { useLocalWorkspaces } from "#/hooks/query/use-local-workspaces";
import { useModelInterceptor } from "#/hooks/chat/use-model-interceptor";
+import { useLlmConfigured } from "#/hooks/use-llm-configured";
import { HOME_PROMPT_DRAFT_KEY } from "#/hooks/chat/use-draft-persistence";
import { useChatAttachmentUpload } from "#/hooks/chat/use-chat-attachment-upload";
import { useConversationStore } from "#/stores/conversation-store";
@@ -46,6 +47,11 @@ export function HomeChatLauncher() {
const { mutate: createConversation, isPending } = useCreateConversation();
const isCreatingElsewhere = useIsCreatingConversation();
const isCreating = isPending || isCreatingElsewhere;
+ const { isConfigured: isLlmConfigured, isLoading: isLlmConfigLoading } =
+ useLlmConfigured();
+ // Block sending entirely when there's no usable LLM; the banner above the
+ // launcher (rendered by the home route) explains it and offers setup.
+ const llmBlocked = !isLlmConfigLoading && !isLlmConfigured;
const { images, files, imagesMarkedUploadAsFile, clearAllFiles } =
useConversationStore();
const { handleUpload } = useChatAttachmentUpload();
@@ -63,6 +69,11 @@ export function HomeChatLauncher() {
const hasAttachments = images.length > 0 || files.length > 0;
if ((!trimmed && !hasAttachments) || isCreating) return;
+ // Safety net: the input is disabled when there's no usable LLM, but never
+ // create a conversation that can't run (it would fail with a cryptic
+ // API-key error on the first turn).
+ if (llmBlocked) return;
+
const attachmentSnapshot = {
images: [...images],
files: [...files],
@@ -195,7 +206,7 @@ export function HomeChatLauncher() {
<CustomChatInput
onSubmit={handleSubmitWithModelGuard}
onFilesPaste={handleUpload}
- disabled={isCreating}
+ disabled={isCreating || llmBlocked}
/>
</div>
diff --git a/src/components/features/settings/llm-profiles/profile-actions-menu.tsx b/src/components/features/settings/llm-profiles/profile-actions-menu.tsx
index bd6016b049..1b56ed36c2 100644
--- a/src/components/features/settings/llm-profiles/profile-actions-menu.tsx
+++ b/src/components/features/settings/llm-profiles/profile-actions-menu.tsx
@@ -12,7 +12,6 @@ import { cn } from "#/utils/utils";
import { dropdownMenuListClassName } from "#/utils/dropdown-classes";
import { I18nKey } from "#/i18n/declaration";
import { ConversationNameContextMenuIconText } from "#/components/features/conversation/conversation-name-context-menu-icon-text";
-import { StyledTooltip } from "#/components/shared/buttons/styled-tooltip";
import EditIcon from "#/icons/u-edit.svg?react";
import CheckCircleIcon from "#/icons/u-check-circle.svg?react";
import DeleteIcon from "#/icons/u-delete.svg?react";
@@ -217,30 +216,17 @@ export function ProfileActionsMenu({
disabled={setActiveDisabled}
testId="profile-set-active"
/>
- {(() => {
- const deleteItem = (
- <MenuItem
- index={3}
- icon={<DeleteIcon width={16} height={16} />}
- label={t(I18nKey.BUTTON$DELETE)}
- onClick={() => handleAction(onDelete)}
- onKeyDown={handleKeyDown}
- menuItemsRef={menuItemsRef}
- disabled={isActive}
- testId="profile-delete"
- />
- );
- if (!isActive) return deleteItem;
- return (
- <StyledTooltip
- content={t(I18nKey.SETTINGS$PROFILE_CANNOT_DELETE_ACTIVE)}
- placement="left"
- >
- {/* span needed so pointer events reach the tooltip trigger on a disabled button */}
- <span className="inline-flex w-full">{deleteItem}</span>
- </StyledTooltip>
- );
- })()}
+ {/* The active profile can be deleted: useEnsureActiveProfile then promotes
+ another remaining profile so a profile is always active in local mode. */}
+ <MenuItem
+ index={3}
+ icon={<DeleteIcon width={16} height={16} />}
+ label={t(I18nKey.BUTTON$DELETE)}
+ onClick={() => handleAction(onDelete)}
+ onKeyDown={handleKeyDown}
+ menuItemsRef={menuItemsRef}
+ testId="profile-delete"
+ />
</div>
);
diff --git a/src/hooks/use-ensure-active-profile.ts b/src/hooks/use-ensure-active-profile.ts
new file mode 100644
index 0000000000..83feb15774
--- /dev/null
+++ b/src/hooks/use-ensure-active-profile.ts
@@ -0,0 +1,51 @@
+import { useEffect, useRef } from "react";
+import { useActiveBackend } from "#/contexts/active-backend-context";
+import { useLlmProfiles } from "#/hooks/query/use-llm-profiles";
+import { useActivateLlmProfile } from "#/hooks/mutation/use-activate-llm-profile";
+
+/**
+ * Local-mode UX policy: keep an LLM profile active whenever at least one
+ * exists, so the agent always has a usable LLM without a manual "activate"
+ * step. This lives in the client on purpose β€” the agent-server is a neutral
+ * API that returns `active_profile` as stored (possibly pointing at a deleted
+ * profile); other consumers (e.g. SaaS) own the LLM differently.
+ *
+ * When profiles exist but none is the active one β€” never activated, or the
+ * active profile was deleted β€” it activates the first profile that has an API
+ * key (falling back to the first profile). With zero profiles it does nothing:
+ * that genuine "no LLM" state is surfaced by the gate/banner.
+ */
+export function useEnsureActiveProfile(): void {
+ const { backend } = useActiveBackend();
+ const isLocal = backend.kind === "local";
+ const { data: profilesData } = useLlmProfiles();
+ const { mutate: activate, isPending } = useActivateLlmProfile();
+
+ // Remember the last profile we tried to activate so we don't re-fire while
+ // the mutation + refetch settle, or hammer a profile whose activation fails.
+ const attemptedRef = useRef<string | null>(null);
+
+ // A backend switch is a clean slate for the above guard.
+ useEffect(() => {
+ attemptedRef.current = null;
+ }, [backend.id]);
+
+ useEffect(() => {
+ if (!isLocal || isPending || !profilesData) return;
+
+ const { profiles, active_profile: activeProfile } = profilesData;
+ const activeValid =
+ activeProfile != null && profiles.some((p) => p.name === activeProfile);
+
+ if (profiles.length === 0 || activeValid) {
+ attemptedRef.current = null;
+ return;
+ }
+
+ // Prefer a profile with a key so the result is immediately usable.
+ const target = profiles.find((p) => p.api_key_set) ?? profiles[0];
+ if (attemptedRef.current === target.name) return;
+ attemptedRef.current = target.name;
+ activate(target.name);
+ }, [isLocal, profilesData, isPending, activate]);
+}
diff --git a/src/hooks/use-llm-configured.ts b/src/hooks/use-llm-configured.ts
index 4264e53760..7ece4dea03 100644
--- a/src/hooks/use-llm-configured.ts
+++ b/src/hooks/use-llm-configured.ts
@@ -1,6 +1,7 @@
import { useSettings } from "#/hooks/query/use-settings";
import { useConfig } from "#/hooks/query/use-config";
import { useLlmProfiles } from "#/hooks/query/use-llm-profiles";
+import { useActiveBackend } from "#/contexts/active-backend-context";
import { isSettingsPageHidden } from "#/utils/settings-utils";
interface LlmConfiguredResult {
@@ -43,6 +44,8 @@ export function useLlmConfigured(): LlmConfiguredResult {
isLoading: profilesLoading,
isError: profilesError,
} = useLlmProfiles();
+ const { backend } = useActiveBackend();
+ const isLocal = backend.kind === "local";
const isAcpAgent = settings?.agent_settings?.agent_kind === "acp";
const hasApiKey = settings?.llm_api_key_set === true;
@@ -55,6 +58,13 @@ export function useLlmConfigured(): LlmConfiguredResult {
config?.feature_flags,
);
+ // In local mode, profiles are the source of truth: a usable LLM must be
+ // backed by an active profile that still exists and has a key. The raw
+ // settings key can be a stale copy left behind by a deleted profile
+ // (settings are not cleared on delete), so we don't count it here. Cloud
+ // backends don't use profiles and keep the settings-key signal.
+ const hasUsableLlm = isLocal ? hasActiveProfileApiKey : hasApiKey;
+
// Treat a fetch failure as indeterminate (same as loading) only when it
// leaves us with no data to decide from β€” otherwise a transient network
// error would surface the banner with the same urgency as a genuinely
@@ -68,8 +78,7 @@ export function useLlmConfigured(): LlmConfiguredResult {
profilesLoading || (profilesError && !profilesData);
return {
- isConfigured:
- isAcpAgent || hasApiKey || hasActiveProfileApiKey || llmSettingsHidden,
+ isConfigured: isAcpAgent || llmSettingsHidden || hasUsableLlm,
isLoading:
settingsIndeterminate || configIndeterminate || profilesIndeterminate,
};
diff --git a/src/routes/root-layout.tsx b/src/routes/root-layout.tsx
index 221e28a8dc..741f084c3b 100644
--- a/src/routes/root-layout.tsx
+++ b/src/routes/root-layout.tsx
@@ -14,6 +14,7 @@ import { SidebarMobileNavProvider } from "#/components/features/sidebar/sidebar-
import { SidebarMobileMenuBar } from "#/components/features/sidebar/sidebar-mobile-menu-bar";
import { useSettings } from "#/hooks/query/use-settings";
import { useMigrateUserConsent } from "#/hooks/use-migrate-user-consent";
+import { useEnsureActiveProfile } from "#/hooks/use-ensure-active-profile";
import { useSyncPostHogConsent } from "#/hooks/use-sync-posthog-consent";
import { usePostHogIdentify } from "#/hooks/use-posthog-identify";
import { LoadingSpinner } from "#/components/shared/loading-spinner";
@@ -83,6 +84,8 @@ export default function MainApp() {
useSyncPostHogConsent();
usePostHogIdentify();
+ // Local-mode policy: keep a profile active so a usable LLM is always selected.
+ useEnsureActiveProfile();
React.useEffect(() => {
if (settings?.language) {
diff --git a/tests/e2e/mock-llm/mock-llm-acp-agent.spec.ts b/tests/e2e/mock-llm/mock-llm-acp-agent.spec.ts
index 1ce30c0563..704f3e3772 100644
--- a/tests/e2e/mock-llm/mock-llm-acp-agent.spec.ts
+++ b/tests/e2e/mock-llm/mock-llm-acp-agent.spec.ts
@@ -85,7 +85,7 @@ test.describe("mock-LLM ACP agent conversation", () => {
await seedLocalStorage(page);
});
- test.afterAll(async ({ request }) => {
+ test.afterAll(async ({ request, browser }) => {
// Clean up the conversation
if (conversationId) {
try {
@@ -103,10 +103,14 @@ test.describe("mock-LLM ACP agent conversation", () => {
} catch {
// best-effort
}
+ const page = await browser.newPage();
try {
- await ensureMockLLMProfile(request);
+ await seedLocalStorage(page);
+ await ensureMockLLMProfile(page);
} catch {
// best-effort
+ } finally {
+ await page.close();
}
try {
await resetMockLLM(request);
@@ -123,8 +127,8 @@ test.describe("mock-LLM ACP agent conversation", () => {
}) => {
// The agent-server may make internal LLM calls (condenser) even for
// ACP conversations. Ensure a mock LLM profile exists so those calls
- // don't fail. This API call is not what we're testing β€” the ACP UI is.
- await ensureMockLLMProfile(request);
+ // don't fail. This UI flow is not what we're testing β€” the ACP UI is.
+ await ensureMockLLMProfile(page);
await routeSessionApiKey(page);
await page.goto("/settings/agent", { waitUntil: "domcontentloaded" });
diff --git a/tests/e2e/mock-llm/mock-llm-automation.spec.ts b/tests/e2e/mock-llm/mock-llm-automation.spec.ts
index 2b06cbb793..7555ae9759 100644
--- a/tests/e2e/mock-llm/mock-llm-automation.spec.ts
+++ b/tests/e2e/mock-llm/mock-llm-automation.spec.ts
@@ -208,10 +208,11 @@ test.describe("mock-LLM automation lifecycle", () => {
// ── Step 1: Ensure LLM profile + register the automation trajectory ─
test("step 1: setup LLM profile and register automation trajectory", async ({
+ page,
request,
}) => {
- // Ensure the mock LLM profile is configured
- await ensureMockLLMProfile(request);
+ // Ensure the mock LLM profile is configured via the Settings UI
+ await ensureMockLLMProfile(page);
// Build the terminal commands the mock LLM will return.
// The curl commands hit the REAL automation backend through the ingress.
diff --git a/tests/e2e/mock-llm/mock-llm-conversation.spec.ts b/tests/e2e/mock-llm/mock-llm-conversation.spec.ts
index 46116f1424..0c1bea8c56 100644
--- a/tests/e2e/mock-llm/mock-llm-conversation.spec.ts
+++ b/tests/e2e/mock-llm/mock-llm-conversation.spec.ts
@@ -173,9 +173,18 @@ test.describe("mock-LLM agent-server conversation", () => {
// Open the actions menu for this profile
await targetRow!.getByTestId("profile-menu-trigger").click();
-
- // Click "Set as active"
- await page.getByTestId("profile-set-active").click();
+ await waitForTestId(page, "profile-actions-menu");
+
+ // Click "Set as active" β€” with client-side reconciliation
+ // (useEnsureActiveProfile) a freshly-created keyed profile may already be
+ // auto-activated, which disables this item. Only click when it isn't active
+ // yet; either way the badge poll below verifies the end state.
+ const setActive = page.getByTestId("profile-set-active");
+ if (await setActive.isEnabled()) {
+ await setActive.click();
+ } else {
+ await page.keyboard.press("Escape");
+ }
// Verify the "Active" badge appears on our profile.
// Poll with reload instead of a fixed timeout β€” the mutation may take
diff --git a/tests/e2e/mock-llm/mock-llm-image-upload.spec.ts b/tests/e2e/mock-llm/mock-llm-image-upload.spec.ts
index a4b71f568c..fa1cc9a1df 100644
--- a/tests/e2e/mock-llm/mock-llm-image-upload.spec.ts
+++ b/tests/e2e/mock-llm/mock-llm-image-upload.spec.ts
@@ -77,13 +77,13 @@ test.describe("mock-LLM image upload", () => {
page,
request,
}) => {
- // ── 1. Configure mock LLM via API (avoids repeating the UI profile steps) ──
+ // ── 1. Configure mock LLM via the Settings UI ──
// Use a vision-capable model name so litellm does not strip image_url
// content blocks when constructing the completion request. The base_url
// still points at the local mock server; the model name is purely a hint
// to litellm about what content types the model accepts.
- await ensureMockLLMProfile(request, "openai/gpt-4o");
+ await ensureMockLLMProfile(page, { model: "openai/gpt-4o" });
// ── 2. Register and activate the trajectory ──
// The mock LLM ignores the request body, so we don't need the agent to
diff --git a/tests/e2e/mock-llm/mock-llm-model-switch.spec.ts b/tests/e2e/mock-llm/mock-llm-model-switch.spec.ts
index 4e67a2c255..f27f8f047f 100644
--- a/tests/e2e/mock-llm/mock-llm-model-switch.spec.ts
+++ b/tests/e2e/mock-llm/mock-llm-model-switch.spec.ts
@@ -74,6 +74,7 @@ async function saveProfile(
api_key: "mock-api-key-for-testing",
base_url: MOCK_LLM_AGENT_URL,
},
+ include_secrets: true,
},
},
);
@@ -128,12 +129,12 @@ test.describe("mock-LLM /model slash command", () => {
// ── Step 1: Configure LLM + create switch-target profile + register trajectory
test("step 1: configure LLM, create switch-target profile, register trajectory", async ({
+ page,
request,
}) => {
- // Use the proven ensureMockLLMProfile helper to set agent_settings.llm
- // so the agent-server can reach the mock LLM. This is the same approach
- // used by mock-llm-conversation.spec.ts.
- await ensureMockLLMProfile(request);
+ // Use the Settings UI to create + activate a mock LLM profile β€” the same
+ // flow used by mock-llm-conversation.spec.ts.
+ await ensureMockLLMProfile(page);
// Create profile B as the switch target β€” it has a different model name
// but the same mock LLM base_url so post-switch inference still works.
diff --git a/tests/e2e/mock-llm/mock-llm-preset-automation.spec.ts b/tests/e2e/mock-llm/mock-llm-preset-automation.spec.ts
index 298b707a9d..9d1be1d39d 100644
--- a/tests/e2e/mock-llm/mock-llm-preset-automation.spec.ts
+++ b/tests/e2e/mock-llm/mock-llm-preset-automation.spec.ts
@@ -149,7 +149,7 @@ test.describe("preset automation β†’ slash command conversation", () => {
page,
request,
}) => {
- await ensureMockLLMProfile(request);
+ await ensureMockLLMProfile(page);
// Configure a dummy Slack MCP server so the card is launchable
await test.step("configure dummy Slack MCP", async () => {
@@ -237,7 +237,7 @@ test.describe("preset automation β†’ slash command conversation", () => {
page,
request,
}) => {
- await ensureMockLLMProfile(request);
+ await ensureMockLLMProfile(page);
// Explicitly clear the MCP config left by test 1.
// Setting mcp_config to null removes it entirely (an empty {} or
diff --git a/tests/e2e/mock-llm/mock-llm-profile-management.spec.ts b/tests/e2e/mock-llm/mock-llm-profile-management.spec.ts
index f186761253..e6a7e9f503 100644
--- a/tests/e2e/mock-llm/mock-llm-profile-management.spec.ts
+++ b/tests/e2e/mock-llm/mock-llm-profile-management.spec.ts
@@ -3,12 +3,12 @@
*
* Covers two scenarios that previously had no end-to-end guard:
*
- * 1. Active profile deletion guard (PR #1127):
- * The "Delete" option must not be available for the active LLM
- * profile. Deleting the active profile leaves the app in an
- * inconsistent state β€” conversations fail with missing settings.
- * The fix adds a disabled-with-tooltip guard in the UI, which
- * this test verifies.
+ * 1. Active profile deletion + reconciliation:
+ * The active LLM profile IS deletable (the PR #1127 disable-guard was
+ * removed). Deleting it must not strand the app: useEnsureActiveProfile
+ * promotes a remaining profile to active in local mode, so a usable LLM
+ * is always selected. This test verifies delete is enabled and that the
+ * remaining profile becomes active.
*
* 2. Same-model profile identity (PR #1123):
* When two profiles share the same underlying model, the chat
@@ -65,6 +65,7 @@ async function saveProfile(
api_key: "mock-api-key-for-testing",
base_url: MOCK_LLM_AGENT_URL,
},
+ include_secrets: true,
},
},
);
@@ -95,7 +96,7 @@ test.describe.configure({ mode: "serial" });
// Test 1 β€” Active profile deletion guard (PR #1127)
// ═══════════════════════════════════════════════════════════════════════
-test.describe("active profile deletion guard", () => {
+test.describe("active profile deletion + reconciliation", () => {
const ACTIVE_PROFILE = "deletion-guard-active";
const INACTIVE_PROFILE = "deletion-guard-inactive";
@@ -113,7 +114,7 @@ test.describe("active profile deletion guard", () => {
}
});
- test("delete is disabled on the active profile and enabled on inactive profiles", async ({
+ test("active profile is deletable and reconciliation activates another profile", async ({
page,
request,
}) => {
@@ -127,20 +128,20 @@ test.describe("active profile deletion guard", () => {
await dismissAnalyticsModal(page);
await waitForTestId(page, "add-llm-profile");
- // ── Find the active profile row and open its actions menu ──
- await test.step("active profile: delete button should be disabled", async () => {
- const profileRows = page.getByTestId("profile-row");
- const rowCount = await profileRows.count();
- let activeRow: ReturnType<typeof profileRows.nth> | null = null;
-
- for (let i = 0; i < rowCount; i++) {
- const row = profileRows.nth(i);
- const text = await row.textContent();
- if (text?.includes(ACTIVE_PROFILE)) {
- activeRow = row;
- break;
- }
+ const rowFor = async (name: string) => {
+ const rows = page.getByTestId("profile-row");
+ const count = await rows.count();
+ for (let i = 0; i < count; i++) {
+ const row = rows.nth(i);
+ if ((await row.textContent())?.includes(name)) return row;
}
+ return null;
+ };
+
+ // ── Delete is now enabled on the active profile (the #1127 guard was
+ // removed; useEnsureActiveProfile keeps a profile active instead) ──
+ await test.step("active profile: delete button is enabled", async () => {
+ const activeRow = await rowFor(ACTIVE_PROFILE);
expect(
activeRow,
`Could not find profile row for "${ACTIVE_PROFILE}"`,
@@ -149,7 +150,6 @@ test.describe("active profile deletion guard", () => {
await activeRow!.getByTestId("profile-menu-trigger").click();
await waitForTestId(page, "profile-actions-menu");
- // The Delete button should be rendered but disabled for the active profile.
const deleteButton = page.getByTestId("profile-delete");
await expect(
deleteButton,
@@ -157,35 +157,19 @@ test.describe("active profile deletion guard", () => {
).toBeVisible();
await expect(
deleteButton,
- "Delete button should be disabled for the active profile",
- ).toBeDisabled();
+ "Delete should be enabled for the active profile",
+ ).toBeEnabled();
// Edit and Set-as-active should still be present
await expect(page.getByTestId("profile-edit")).toBeVisible();
await expect(page.getByTestId("profile-set-active")).toBeVisible();
- // Close the menu
await page.keyboard.press("Escape");
});
- // ── Find an inactive profile and verify delete is enabled ──
- await test.step("inactive profile: delete button should be enabled", async () => {
- // Reload to reset any stale menu state
- await page.goto("/settings/llm", { waitUntil: "domcontentloaded" });
- await waitForTestId(page, "add-llm-profile");
-
- const profileRows = page.getByTestId("profile-row");
- const rowCount = await profileRows.count();
- let inactiveRow: ReturnType<typeof profileRows.nth> | null = null;
-
- for (let i = 0; i < rowCount; i++) {
- const row = profileRows.nth(i);
- const text = await row.textContent();
- if (text?.includes(INACTIVE_PROFILE)) {
- inactiveRow = row;
- break;
- }
- }
+ // ── Delete is enabled on an inactive profile too ──
+ await test.step("inactive profile: delete button is enabled", async () => {
+ const inactiveRow = await rowFor(INACTIVE_PROFILE);
expect(
inactiveRow,
`Could not find profile row for "${INACTIVE_PROFILE}"`,
@@ -193,18 +177,51 @@ test.describe("active profile deletion guard", () => {
await inactiveRow!.getByTestId("profile-menu-trigger").click();
await waitForTestId(page, "profile-actions-menu");
+ await expect(page.getByTestId("profile-delete")).toBeEnabled();
+ await page.keyboard.press("Escape");
+ });
- const deleteButton = page.getByTestId("profile-delete");
- await expect(
- deleteButton,
- "Delete button should be present for inactive profile",
- ).toBeVisible();
- await expect(
- deleteButton,
- "Delete button should be enabled for inactive profiles",
- ).toBeEnabled();
+ // ── Deleting the active profile reconciles to the remaining one ──
+ await test.step("deleting the active profile activates the remaining profile", async () => {
+ const activeRow = await rowFor(ACTIVE_PROFILE);
+ expect(activeRow).not.toBeNull();
- await page.keyboard.press("Escape");
+ await activeRow!.getByTestId("profile-menu-trigger").click();
+ await waitForTestId(page, "profile-actions-menu");
+ await page.getByTestId("profile-delete").click();
+
+ // Confirm in the delete modal.
+ await page.getByTestId("delete-profile-confirm").click();
+
+ // useEnsureActiveProfile re-activates the only remaining profile. Poll
+ // with reload β€” the delete + activate mutations may take a moment on CI.
+ await expect
+ .poll(
+ async () => {
+ await page.goto("/settings/llm", {
+ waitUntil: "domcontentloaded",
+ });
+ await waitForTestId(page, "add-llm-profile");
+ const remaining = await rowFor(INACTIVE_PROFILE);
+ if (!remaining) return false;
+ // The deleted profile must be gone, and reconciliation must keep
+ // *some* profile active (the "always have an active profile"
+ // guarantee). We don't assert it's INACTIVE_PROFILE specifically β€”
+ // other profiles may linger on the shared agent-server and
+ // useEnsureActiveProfile activates the first keyed one.
+ const goneRow = await rowFor(ACTIVE_PROFILE);
+ const activeBadges = await page
+ .getByTestId("profile-active-badge")
+ .count();
+ return goneRow === null && activeBadges > 0;
+ },
+ {
+ message: `"${INACTIVE_PROFILE}" should become active after deleting "${ACTIVE_PROFILE}"`,
+ timeout: 15_000,
+ intervals: [1_000, 2_000, 3_000],
+ },
+ )
+ .toBe(true);
});
});
});
diff --git a/tests/e2e/mock-llm/reporters/done-marker-reporter.ts b/tests/e2e/mock-llm/reporters/done-marker-reporter.ts
index 701e410cda..f131dc5924 100644
--- a/tests/e2e/mock-llm/reporters/done-marker-reporter.ts
+++ b/tests/e2e/mock-llm/reporters/done-marker-reporter.ts
@@ -1,17 +1,16 @@
/**
- * Custom Playwright reporter that writes a marker file when all tests
- * complete β€” before webServer teardown starts.
+ * Custom Playwright reporter that writes marker files for CI coordination.
*
- * This lets the CI wrapper detect test completion and kill the hanging
- * teardown process immediately, instead of waiting for a timeout.
- *
- * Marker files are written to a `.mock-llm-markers/` directory at the
- * project root β€” intentionally outside Playwright's `outputDir`
- * (`test-results-mock-llm/`) to avoid being cleaned up.
+ * Marker files are written to `.mock-llm-markers/` at the project root β€”
+ * intentionally outside Playwright's `outputDir` (`test-results-mock-llm/`)
+ * to avoid being cleaned up.
*
* Written markers:
- * .tests-done β€” always written; content is "passed" or "failed"
- * .all-passed β€” written only when all tests passed
+ * .results.json β€” written after EVERY test; always has the latest results
+ * so that even a mid-suite kill leaves usable data
+ * .tests-done β€” written only when all tests complete; content is
+ * "passed" or "failed"
+ * .all-passed β€” written only when all tests passed
*/
import { mkdirSync, writeFileSync } from "node:fs";
@@ -34,25 +33,22 @@ interface TestRecord {
}
/**
- * Tracks test results and writes markers at the earliest possible moment.
- *
- * Playwright lifecycle: onBegin β†’ tests β†’ onTestEnd (each) β†’ onEnd β†’ cleanup.
- * The webServer teardown happens during "cleanup", which can hang indefinitely.
- * `onEnd()` fires AFTER teardown, so it's too late.
+ * Tracks test results and writes them incrementally.
*
- * Instead, we write the `.tests-done` marker in `onTestEnd()` after the LAST
- * test completes. For a single-test suite this fires immediately after the
- * test finishes, before any webServer teardown begins.
+ * `.results.json` is flushed after every `onTestEnd()` so the CI report
+ * script always has data β€” even when the process is killed mid-suite
+ * (e.g. the CI polling deadline expires before all tests finish).
*
- * Also writes `.results.json` with per-test timing/error data so the report
- * script can render accurate durations (Playwright's own results.json only
- * flushes on clean exit, which never happens when teardown hangs).
+ * `.tests-done` / `.all-passed` are written only when the full suite
+ * completes, letting the CI wrapper distinguish "still running" from
+ * "done".
*/
class DoneMarkerReporter implements Reporter {
private totalTests = 0;
private completedTests = 0;
private allPassed = true;
private tests: TestRecord[] = [];
+ private markerDirCreated = false;
onBegin(_config: unknown, suite: { allTests(): TestCase[] }) {
this.totalTests = suite.allTests().length;
@@ -76,39 +72,69 @@ class DoneMarkerReporter implements Reporter {
.slice(0, 1500),
});
- // Write markers after the last test completes.
+ // Always flush results so a mid-suite kill still leaves usable data.
+ this.writeResults();
+
+ // Write completion markers only after the last test.
if (this.completedTests >= this.totalTests) {
- this.writeMarkers();
+ this.writeCompletionMarkers();
}
}
onEnd(_result: FullResult) {
- // Fallback: write markers here too in case onTestEnd didn't fire
- // (e.g., if Playwright exits before running any tests).
- // If zero tests ran (webServer timeout, config error, etc.), treat
- // that as a failure β€” "0 tests passed" is not a meaningful pass.
+ // Fallback: if onTestEnd never fired (webServer timeout, config
+ // error, etc.), treat that as a failure and write what we have.
if (this.totalTests === 0 || this.completedTests === 0) {
this.allPassed = false;
}
- this.writeMarkers();
+ this.writeResults();
+ this.writeCompletionMarkers();
+ }
+
+ /** Flush per-test timing/error data β€” called after every test. */
+ private writeResults() {
+ const done = this.completedTests >= this.totalTests;
+ const status = done
+ ? this.allPassed
+ ? "passed"
+ : "failed"
+ : "in_progress";
+ try {
+ this.ensureMarkerDir();
+ writeFileSync(
+ join(MARKER_DIR, ".results.json"),
+ JSON.stringify({
+ status,
+ completed: this.completedTests,
+ total: this.totalTests,
+ tests: this.tests,
+ }),
+ );
+ } catch {
+ // Don't crash Playwright if marker write fails
+ }
}
- private writeMarkers() {
+ /** Write .tests-done and .all-passed β€” only when the suite is complete. */
+ private writeCompletionMarkers() {
const status = this.allPassed ? "passed" : "failed";
try {
- mkdirSync(MARKER_DIR, { recursive: true });
+ this.ensureMarkerDir();
writeFileSync(join(MARKER_DIR, ".tests-done"), status);
if (this.allPassed) {
writeFileSync(join(MARKER_DIR, ".all-passed"), "1");
}
- writeFileSync(
- join(MARKER_DIR, ".results.json"),
- JSON.stringify({ status, tests: this.tests }),
- );
} catch {
// Don't crash Playwright if marker write fails
}
}
+
+ private ensureMarkerDir() {
+ if (!this.markerDirCreated) {
+ mkdirSync(MARKER_DIR, { recursive: true });
+ this.markerDirCreated = true;
+ }
+ }
}
export default DoneMarkerReporter;
diff --git a/tests/e2e/mock-llm/scripts/render-mock-llm-report.mjs b/tests/e2e/mock-llm/scripts/render-mock-llm-report.mjs
index e8c3e420ef..31eb8fd85b 100644
--- a/tests/e2e/mock-llm/scripts/render-mock-llm-report.mjs
+++ b/tests/e2e/mock-llm/scripts/render-mock-llm-report.mjs
@@ -141,7 +141,7 @@ function overallIcon(status) {
// ── Report rendering ───────────────────────────────────────────────────
-function renderReport({ tests, workflowUrl, commit, artifactUrl, title }) {
+function renderReport({ tests, workflowUrl, commit, artifactUrl, title, markerMeta }) {
const status = overallStatus(tests);
const icon = overallIcon(status);
const passed = tests.filter((t) => t.status === "passed").length;
@@ -150,17 +150,24 @@ function renderReport({ tests, workflowUrl, commit, artifactUrl, title }) {
).length;
const skipped = tests.filter((t) => t.status === "skipped").length;
const total = tests.length;
+ const wasKilledMidSuite =
+ markerMeta?.status === "in_progress" && markerMeta.total > markerMeta.completed;
const lines = [];
- // Header
- lines.push(`## ${icon} ${title || "Mock-LLM E2E Tests"}`);
+ // Header β€” use πŸ›‘ when killed mid-suite so it's visually distinct
+ const headerIcon = wasKilledMidSuite ? "πŸ›‘" : icon;
+ lines.push(`## ${headerIcon} ${title || "Mock-LLM E2E Tests"}`);
lines.push("");
// Summary line
const parts = [`**${passed}/${total} passed**`];
if (failed) parts.push(`**${failed} failed**`);
if (skipped) parts.push(`${skipped} skipped`);
+ if (wasKilledMidSuite) {
+ const notRun = markerMeta.total - markerMeta.completed;
+ parts.push(`⚠️ **${notRun} not run** (process killed at ${markerMeta.completed}/${markerMeta.total})`);
+ }
lines.push(parts.join(" Β· "));
lines.push("");
@@ -223,16 +230,19 @@ const outputPath = args.output || "mock-llm-report.md";
const data = loadResults(resultsPath);
let tests = data ? collectTests(data.suites) : [];
-// When Playwright is killed during webServer teardown, the JSON reporter
-// never flushes results.json. Fall back to .results.json written by
-// DoneMarkerReporter (onTestEnd) which fires before teardown.
+// When Playwright is killed during webServer teardown (or mid-suite),
+// the JSON reporter never flushes results.json. Fall back to .results.json
+// written incrementally by DoneMarkerReporter after every onTestEnd().
+let markerMeta = null;
if (!data || tests.length === 0) {
const markerDir = args.marker_dir || ".mock-llm-markers";
const markerResultsPath = `${markerDir}/.results.json`;
const donePath = `${markerDir}/.tests-done`;
if (existsSync(markerResultsPath)) {
- // Rich results from DoneMarkerReporter β€” has per-test timing & errors
+ // Rich results from DoneMarkerReporter β€” has per-test timing & errors.
+ // May be partial (status: "in_progress") if the process was killed
+ // before all tests finished.
const markerData = JSON.parse(readFileSync(markerResultsPath, "utf8"));
tests = (markerData.tests ?? []).map((t) => ({
title: t.title,
@@ -241,8 +251,13 @@ if (!data || tests.length === 0) {
retryCount: 0,
error: t.error ?? "",
}));
+ markerMeta = {
+ status: markerData.status,
+ completed: markerData.completed ?? tests.length,
+ total: markerData.total ?? tests.length,
+ };
console.log(
- `No results.json; using marker results (${tests.length} tests, ${markerData.status})`,
+ `No results.json; using marker results (${tests.length} tests run, ${markerMeta.completed}/${markerMeta.total} completed, status: ${markerData.status})`,
);
} else if (existsSync(donePath)) {
// Minimal fallback β€” just pass/fail status, no timing
@@ -275,6 +290,7 @@ const report = renderReport({
commit: args.commit || "",
artifactUrl: args.artifact_url || "",
title: args.title || "",
+ markerMeta,
});
writeFileSync(outputPath, report);
diff --git a/tests/e2e/mock-llm/utils/mock-llm-helpers.ts b/tests/e2e/mock-llm/utils/mock-llm-helpers.ts
index 827002ff4a..a3051088ce 100644
--- a/tests/e2e/mock-llm/utils/mock-llm-helpers.ts
+++ b/tests/e2e/mock-llm/utils/mock-llm-helpers.ts
@@ -133,12 +133,20 @@ export async function waitForTestId(
/** Dismiss the analytics consent modal if it appears. */
export async function dismissAnalyticsModal(page: Page) {
await page.waitForLoadState("domcontentloaded");
- // Quick check β€” if the modal is there, click "Confirm preferences"
+ // The analytics consent modal is lazy-loaded via React.Suspense and may
+ // appear several seconds after DOM-content-loaded. Wait for the form's
+ // test ID (more specific than the generic ModalBackdrop) or give up after
+ // a generous window so tests that have already dismissed the modal don't
+ // stall. Also dismiss any other ModalBackdrop dialog that may block
+ // pointer events (e.g. the delete-profile confirmation left by a prior
+ // test).
try {
- const confirmButton = page.getByRole("button", {
- name: "Confirm preferences",
- });
- await confirmButton.click({ timeout: 3_000 });
+ const form = page.getByTestId("user-capture-consent-form");
+ await form.waitFor({ state: "visible", timeout: 5_000 });
+ await form.getByRole("button", { name: "Confirm preferences" }).click();
+ // Wait for the modal to fully close so the backdrop no longer
+ // intercepts pointer events.
+ await form.waitFor({ state: "hidden", timeout: 5_000 });
} catch {
// Modal didn't appear β€” that's fine
}
@@ -318,61 +326,137 @@ export async function deleteConversation(
// ═══════════════════════════════════════════════════════════════════════
/**
- * Ensure the currently-active LLM profile is configured to point at the mock
- * LLM server.
- *
- * This bypasses the UI-driven profile creation (steps 1+2 in the conversation
- * test) and directly PATCHes the agent-server settings on whatever profile is
- * currently active. Useful for tests that need to run independently of the
- * conversation test's UI-based profile setup.
+ * Create (or overwrite) an LLM profile and activate it through the Settings
+ * UI β€” the same flow a real user follows.
*
- * Note: this does NOT create or switch profiles β€” it only configures the
- * current profile's LLM settings. Profile creation/activation is handled
- * by the conversation test's UI flow or by the agent-server's default
- * profile behavior.
+ * Exercises the full frontend save path (including `include_secrets`) so the
+ * api_key is persisted correctly. Callers that previously used the API-only
+ * `ensureMockLLMProfile(request)` should switch to this.
*/
export async function ensureMockLLMProfile(
- request: APIRequestContext,
- model = "openai/mock-test-model",
+ page: Page,
+ {
+ profileName = "mock-llm",
+ model = "openai/mock-test-model",
+ apiKey = "mock-api-key-for-testing",
+ baseUrl = MOCK_LLM_AGENT_URL,
+ }: {
+ profileName?: string;
+ model?: string;
+ apiKey?: string;
+ baseUrl?: string;
+ } = {},
) {
- // Check if the current profile already has the mock LLM settings.
- // Use MOCK_LLM_AGENT_URL β€” this is the URL the agent-server will use to
- // reach the mock LLM, which may differ from MOCK_LLM_BASE_URL in Docker.
- const settingsResp = await request.get(`${BACKEND_URL}/api/settings`, {
- headers: {
- "X-Session-API-Key": SESSION_API_KEY,
- "X-Expose-Secrets": "encrypted",
- },
- });
+ await routeSessionApiKey(page);
+ await page.goto("/settings/llm", { waitUntil: "domcontentloaded" });
+ await dismissAnalyticsModal(page);
+ await waitForTestId(page, "add-llm-profile");
+
+ // If a profile with this name already exists, delete it first so we
+ // always start clean (other tests may have left stale config).
+ await deleteProfileIfExists(page, profileName);
+
+ // ── Create the profile ──────────────────────────────────────────────
+ await page.getByTestId("add-llm-profile").click();
+ await waitForTestId(page, "profile-editor-title");
- if (settingsResp.ok()) {
- const settings = await settingsResp.json();
- const llm = settings?.agent_settings?.llm;
- if (llm?.model === model && llm?.base_url === MOCK_LLM_AGENT_URL) {
- return; // Already configured
+ const nameInput = page.getByTestId("profile-name-input");
+ await nameInput.click();
+ await nameInput.fill(profileName);
+
+ // Switch to "All" view so base_url is visible
+ await page.getByTestId("sdk-section-all-toggle").click();
+ await waitForTestId(page, "llm-settings-form-advanced");
+
+ const modelInput = page.getByTestId("llm-custom-model-input");
+ await modelInput.click();
+ await modelInput.fill(model);
+
+ const baseUrlInput = page.getByTestId("base-url-input");
+ await baseUrlInput.click();
+ await baseUrlInput.fill(baseUrl);
+
+ const apiKeyInput = page.getByTestId("llm-api-key-input");
+ await apiKeyInput.click();
+ await apiKeyInput.fill(apiKey);
+
+ await page.getByTestId("save-profile-btn").click();
+ await waitForTestId(page, "add-llm-profile");
+
+ // ── Activate the profile ────────────────────────────────────────────
+ await activateProfileViaUI(page, profileName);
+}
+
+/**
+ * Delete a profile by name through the Settings UI if it exists.
+ * Assumes the page is already on /settings/llm with profiles loaded.
+ */
+async function deleteProfileIfExists(page: Page, profileName: string) {
+ // Use the profile name span's `title` attribute for exact matching
+ // to avoid substring collisions (e.g. "mock-llm" vs "mock-llm-e2e").
+ const row = page
+ .getByTestId("profile-row")
+ .filter({ has: page.locator(`span[title="${profileName}"]`) })
+ .first();
+ if ((await row.count()) === 0) return;
+
+ await row.getByTestId("profile-menu-trigger").click();
+ await waitForTestId(page, "profile-actions-menu");
+ const deleteBtn = page.getByTestId("profile-delete");
+ if (await deleteBtn.isVisible()) {
+ await deleteBtn.click();
+ // Confirm the deletion dialog (test ID: delete-profile-confirm)
+ const confirmBtn = page.getByTestId("delete-profile-confirm");
+ await confirmBtn.waitFor({ state: "visible", timeout: 5_000 });
+ await confirmBtn.click();
+ await waitForTestId(page, "add-llm-profile");
+ } else {
+ await page.keyboard.press("Escape");
+ }
+}
+
+/**
+ * Activate a profile by name through the Settings UI.
+ * Assumes the page is already on /settings/llm with profiles loaded.
+ * Polls until the "Active" badge is visible on the target profile row.
+ */
+async function activateProfileViaUI(page: Page, profileName: string) {
+ const exactRow = (p: Page) =>
+ p
+ .getByTestId("profile-row")
+ .filter({ has: p.locator(`span[title="${profileName}"]`) })
+ .first();
+
+ const row = exactRow(page);
+ if ((await row.count()) > 0) {
+ await row.getByTestId("profile-menu-trigger").click();
+ await waitForTestId(page, "profile-actions-menu");
+ const setActive = page.getByTestId("profile-set-active");
+ if (await setActive.isEnabled()) {
+ await setActive.click();
+ } else {
+ // Already active
+ await page.keyboard.press("Escape");
}
}
- // Configure the active profile's LLM settings
- const patchResp = await request.patch(`${BACKEND_URL}/api/settings`, {
- headers: {
- "X-Session-API-Key": SESSION_API_KEY,
- "Content-Type": "application/json",
- },
- data: {
- agent_settings_diff: {
- llm: {
- model,
- api_key: "mock-api-key-for-testing",
- base_url: MOCK_LLM_AGENT_URL,
- },
+ // Poll until the active badge appears on our profile
+ await expect
+ .poll(
+ async () => {
+ await page.goto("/settings/llm", { waitUntil: "domcontentloaded" });
+ await waitForTestId(page, "add-llm-profile");
+ const target = exactRow(page);
+ if ((await target.count()) === 0) return false;
+ return (await target.getByTestId("profile-active-badge").count()) > 0;
},
- },
- });
- expect(
- patchResp.ok(),
- `PATCH /api/settings failed: ${patchResp.status()}`,
- ).toBe(true);
+ {
+ message: `Profile "${profileName}" should have an "Active" badge`,
+ timeout: 15_000,
+ intervals: [1_000, 2_000, 3_000],
+ },
+ )
+ .toBe(true);
}
/**
diff --git a/tests/e2e/snapshots/collapsible-thinking.snapshot.spec.ts b/tests/e2e/snapshots/collapsible-thinking.snapshot.spec.ts
index 1e3719f811..c9e096807c 100644
--- a/tests/e2e/snapshots/collapsible-thinking.snapshot.spec.ts
+++ b/tests/e2e/snapshots/collapsible-thinking.snapshot.spec.ts
@@ -197,6 +197,32 @@ async function navigateToConversation(page: Page, events: unknown[]) {
});
});
+ // The GUI now hides ChatSuggestions (and shows the "LLM not configured"
+ // banner) unless an active profile with a key exists β€” see useLlmConfigured.
+ // Seed one so the empty conversation renders normally: it restores the
+ // "Let's start building!" readiness signal below and keeps the banner out of
+ // the captured screenshots (matching the baselines).
+ await page.route(
+ (url) => url.pathname.endsWith("/api/profiles"),
+ async (route) => {
+ await route.fulfill({
+ status: 200,
+ contentType: "application/json",
+ body: JSON.stringify({
+ profiles: [
+ {
+ name: "mock",
+ model: "anthropic/claude-sonnet-4-20250514",
+ base_url: null,
+ api_key_set: true,
+ },
+ ],
+ active_profile: "mock",
+ }),
+ });
+ },
+ );
+
await stubWebSocket(page);
await page.goto(`/conversations/${CONVERSATION_ID}`, {