Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Consolidate Metabot e2e tests (#83567)

Chris Vandoodewaard70d5cd55e734
diff --git a/e2e/support/helpers/e2e-metabot-helpers.ts b/e2e/support/helpers/e2e-metabot-helpers.ts
index 1501412e986..5cbd492efd6 100644
--- a/e2e/support/helpers/e2e-metabot-helpers.ts
+++ b/e2e/support/helpers/e2e-metabot-helpers.ts
@@ -42,10 +42,6 @@ export function setupAnthropicLlmProvider({
]);
}
-export function clearLlmProviders() {
- return setLlmProviders([]);
-}
-
export function metabotChatSidebar() {
return cy.findByTestId("metabot-chat");
}
diff --git a/e2e/test/scenarios/metabot/ai-controls.cy.spec.ts b/e2e/test/scenarios/metabot/ai-controls.cy.spec.ts
index 02c365dc489..f9023004470 100644
--- a/e2e/test/scenarios/metabot/ai-controls.cy.spec.ts
+++ b/e2e/test/scenarios/metabot/ai-controls.cy.spec.ts
@@ -1,7 +1,4 @@
-import {
- ALL_USERS_GROUP_ID,
- NORMAL_USER_ID,
-} from "e2e/support/cypress_sample_instance_data";
+import { NORMAL_USER_ID } from "e2e/support/cypress_sample_instance_data";
const { H } = cy;
@@ -19,7 +16,7 @@ describe("AI Controls > Metabot access and customization", () => {
});
describe("Feature Access page", () => {
- it("should display the AI feature access table with groups and save a permission change", () => {
+ it("should display the AI feature access table with groups, save a permission change, and hide the Metabot chat icon for users in a group without Metabot access", () => {
cy.intercept("GET", "/api/ee/ai-controls/permissions").as(
"getPermissions",
);
@@ -61,9 +58,34 @@ describe("AI Controls > Metabot access and customization", () => {
cy.findAllByRole("checkbox").each(($checkbox) => {
cy.wrap($checkbox).should("not.be.checked");
});
+ });
+
+ cy.intercept("GET", "/api/metabot/permissions/user-permissions").as(
+ "getUserPermissions",
+ );
+
+ // Sign in as a normal user (who is only in the All Users group)
+ cy.signInAsNormalUser();
+ cy.visit("/");
+ cy.wait("@getUserPermissions")
+ .its("response.body.permissions.metabot")
+ .should("eq", "no");
+
+ // Wait for the navigation bar to be present
+ cy.findByLabelText("Navigation bar").should("be.visible");
+
+ // The Metabot chat icon should not be present
+ H.appBar()
+ .findByRole("button", { name: /Chat with/ })
+ .should("not.exist");
+ cy.signInAsAdmin();
+ cy.visit("/admin/metabot/usage-controls/ai-feature-access");
+ cy.wait("@getPermissions");
+
+ cy.findByRole("row", { name: /All Users permissions/ }).within(() => {
// Toggle the metabot AI features switch on again
- cy.findByRole("switch").click({ force: true });
+ cy.findByRole("switch").should("not.be.checked").click({ force: true });
cy.wait("@updatePermissions")
.its("response.statusCode")
@@ -74,50 +96,35 @@ describe("AI Controls > Metabot access and customization", () => {
cy.wrap($checkbox).should("be.checked");
});
});
- });
- });
-
- describe("Group access controls", () => {
- it("should not show the Metabot chat icon for users in a group without Metabot access", () => {
- // First, get the current permissions so we can update only the All Users group
- cy.request("GET", "/api/ee/ai-controls/permissions").then((response) => {
- const currentPermissions: Array<{
- group_id: number;
- perm_type: string;
- perm_value: string;
- }> = response.body.permissions;
-
- // Set All Users group's metabot permission to "no"
- const updatedPermissions = currentPermissions.map((p) => {
- if (
- p.group_id === ALL_USERS_GROUP_ID &&
- p.perm_type === "permission/metabot"
- ) {
- return { ...p, perm_value: "no" };
- }
- return p;
- });
- cy.request("PUT", "/api/ee/ai-controls/permissions", {
- permissions: updatedPermissions,
- });
- });
-
- // Sign in as a normal user (who is only in the All Users group)
+ cy.log(
+ "a user in a group with Metabot access sees the Metabot chat icon",
+ );
+ cy.intercept("GET", "/api/metabot/permissions/user-permissions").as(
+ "getNormalUserPermissions",
+ );
cy.signInAsNormalUser();
cy.visit("/");
-
- // Wait for the navigation bar to be present
- cy.findByLabelText("Navigation bar").should("be.visible");
-
- // The Metabot chat icon should not be present
- H.appBar().find('[aria-label*="Chat with"]').should("not.exist");
+ cy.wait("@getNormalUserPermissions")
+ .its("response.body.permissions.metabot")
+ .should("eq", "yes");
+ H.appBar()
+ .findByRole("button", { name: /Chat with/ })
+ .should("be.visible");
});
});
describe("Customization", () => {
- it("should save a custom Metabot name", () => {
+ it("should save a custom Metabot name, icon and illustrations setting and use them in the app", () => {
+ H.updateEnterpriseSettings({
+ "metabot-show-illustrations": true,
+ });
+
cy.intercept("PUT", "/api/setting/metabot-name").as("saveName");
+ cy.intercept("PUT", "/api/setting/metabot-icon").as("saveIcon");
+ cy.intercept("PUT", "/api/setting/metabot-show-illustrations").as(
+ "saveIllustrations",
+ );
cy.visit("/admin/metabot/customization");
@@ -132,16 +139,6 @@ describe("AI Controls > Metabot access and customization", () => {
// Wait for debounced save
cy.wait("@saveName").its("response.statusCode").should("eq", 204);
- // Reload and verify persistence
- cy.reload();
- cy.findByLabelText("AI agent's name").should("have.value", "HAL 9000");
- });
-
- it("should upload a custom Metabot icon", () => {
- cy.intercept("PUT", "/api/setting/metabot-icon").as("saveIcon");
-
- cy.visit("/admin/metabot/customization");
-
H.main().findByText("AI agent's icon").should("be.visible");
cy.findByRole("button", { name: "Upload a custom icon" }).should(
"be.visible",
@@ -150,10 +147,7 @@ describe("AI Controls > Metabot access and customization", () => {
// Upload a tiny PNG via the hidden file input
cy.get('input[type="file"]').selectFile(
{
- contents: Cypress.Buffer.from(
- "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==",
- "base64",
- ),
+ contents: Cypress.Buffer.from(TINY_PNG_BASE64, "base64"),
fileName: "metabot-icon.png",
mimeType: "image/png",
},
@@ -164,19 +158,6 @@ describe("AI Controls > Metabot access and customization", () => {
// The "Remove custom icon" button should be visible
cy.findByLabelText("Remove custom icon").should("be.visible");
- });
-
- it("should hide Metabot illustrations when the toggle is switched off", () => {
- H.updateEnterpriseSettings({
- "metabot-show-illustrations": true,
- });
- H.updateSetting("metabot-enabled?", true);
-
- cy.intercept("PUT", "/api/setting/metabot-show-illustrations").as(
- "saveIllustrations",
- );
-
- cy.visit("/admin/metabot/customization");
H.main().findByText("Metabot illustrations").should("be.visible");
@@ -196,9 +177,23 @@ describe("AI Controls > Metabot access and customization", () => {
.its("response.statusCode")
.should("eq", 204);
- // Navigate to the home page and open Metabot chat to verify illustrations are hidden
cy.visit("/");
- H.openMetabotViaSearchButton();
+ cy.findByLabelText("Navigation bar").should("be.visible");
+
+ // The app bar button tooltip/aria-label should reflect the custom name
+ H.appBar()
+ .findByRole("button", { name: /Chat with HAL 9000/ })
+ .should("be.visible")
+ .within(() => {
+ // When a custom icon is set, MetabotIcon renders an <img> with alt = metabotName
+ cy.get("img").should("be.visible");
+ });
+
+ // Open Metabot chat to verify illustrations are hidden
+ H.appBar()
+ .findByRole("button", { name: /Chat with HAL 9000/ })
+ .click();
+ H.assertChatVisibility("visible");
cy.findByTestId("metabot-empty-chat-info").should("be.visible");
// The SVG illustration should NOT be rendered when showIllustrations=false
@@ -210,40 +205,16 @@ describe("AI Controls > Metabot access and customization", () => {
.findByText(/Explore your metrics and models with AI/)
.should("be.visible");
});
-
- it("should show the custom Metabot name in the app bar button tooltip", () => {
- H.updateSetting("metabot-name", "Aria");
-
- cy.visit("/");
- cy.findByLabelText("Navigation bar").should("be.visible");
-
- // The app bar button tooltip/aria-label should reflect the custom name
- H.appBar()
- .findByRole("button", { name: /Chat with Aria/ })
- .should("be.visible");
- });
-
- it("should show a custom Metabot icon in the app bar when metabot-icon is set", () => {
- H.updateEnterpriseSettings({ "metabot-icon": TINY_PNG_DATA_URI });
-
- cy.visit("/");
- cy.findByLabelText("Navigation bar").should("be.visible");
-
- // The custom icon img should be rendered (alt = metabot name)
- H.appBar()
- .findByRole("button", { name: /Chat with/ })
- .within(() => {
- // When a custom icon is set, MetabotIcon renders an <img> with alt = metabotName
- cy.get("img").should("be.visible");
- });
- });
});
describe("System Prompts pages", () => {
- it("should save a custom Metabot chat system prompt", () => {
+ it("should save custom Metabot chat and SQL generation system prompts", () => {
cy.intercept("PUT", "/api/setting/metabot-chat-system-prompt").as(
"savePrompt",
);
+ cy.intercept("PUT", "/api/setting/metabot-sql-system-prompt").as(
+ "saveSqlPrompt",
+ );
cy.visit("/admin/metabot/system-prompts/metabot-chat");
@@ -260,19 +231,9 @@ describe("AI Controls > Metabot access and customization", () => {
cy.wait("@savePrompt").its("response.statusCode").should("eq", 204);
- // Reload and verify persistence
- cy.reload();
- cy.findByRole("textbox", {
- name: /AI chat prompt instructions/,
- }).should("have.text", "Be concise and helpful.");
- });
-
- it("should save a custom SQL generation system prompt", () => {
- cy.intercept("PUT", "/api/setting/metabot-sql-system-prompt").as(
- "saveSqlPrompt",
- );
-
- cy.visit("/admin/metabot/system-prompts/sql-generation");
+ cy.findByTestId("admin-layout-sidebar")
+ .findByRole("link", { name: "SQL generation" })
+ .click();
cy.findByRole("heading", {
name: "SQL generation prompt instructions",
@@ -289,10 +250,18 @@ describe("AI Controls > Metabot access and customization", () => {
cy.wait("@saveSqlPrompt").its("response.statusCode").should("eq", 204);
+ // Reload and verify persistence
cy.reload();
cy.findByRole("textbox", {
name: /SQL generation prompt instructions/,
}).should("have.text", "Always use uppercase SQL keywords.");
+
+ cy.findByTestId("admin-layout-sidebar")
+ .findByRole("link", { name: "AI chat" })
+ .click();
+ cy.findByRole("textbox", {
+ name: /AI chat prompt instructions/,
+ }).should("have.text", "Be concise and helpful.");
});
});
});
@@ -311,7 +280,7 @@ describe("AI controls > AI usage limits", () => {
llmMockServerTeardown();
});
- describe("AI Limits settings can be saved properly", () => {
+ describe("Instance limit settings and enforcement", () => {
beforeEach(() => {
cy.intercept("GET", "/api/ee/ai-controls/usage/instance").as(
"getInstanceLimit",
@@ -333,10 +302,22 @@ describe("AI controls > AI usage limits", () => {
);
});
- it("should save limit type, reset period, instance limit, and quota-reached message when changed", () => {
+ it("should clear and save instance limit settings, then enforce a zero limit in chat", () => {
+ // Pre-set a limit so there's something to clear
+ cy.request("PUT", "/api/ee/ai-controls/usage/instance", {
+ max_usage: 100,
+ });
+
cy.visit(AI_USAGE_LIMITS_URL);
cy.wait("@getInstanceLimit");
+ cy.findByRole("textbox", {
+ name: "Total monthly instance token limit",
+ }).clear();
+ cy.wait("@updateInstanceLimit").then(({ request }) => {
+ expect(request.body).to.deep.equal({ max_usage: null });
+ });
+
// Change limit type to messages
cy.findByRole("radio", { name: "By message count" }).click({
force: true,
@@ -366,68 +347,29 @@ describe("AI controls > AI usage limits", () => {
value: "You have hit the AI usage limit.",
});
});
- });
-
- it("should save instance limit as null when the field is cleared", () => {
- // Pre-set a limit so there's something to clear
- cy.request("PUT", "/api/ee/ai-controls/usage/instance", {
- max_usage: 100,
- });
-
- cy.visit(AI_USAGE_LIMITS_URL);
- cy.wait("@getInstanceLimit");
-
- cy.findByRole("textbox", {
- name: "Total monthly instance token limit",
- }).clear();
+ cy.log("Enforce a zero instance limit in a fresh chat");
+ cy.findByLabelText("Total weekly instance message limit").type(
+ "{selectall}0",
+ );
cy.wait("@updateInstanceLimit").then(({ request }) => {
- expect(request.body).to.deep.equal({ max_usage: null });
+ expect(request.body).to.deep.equal({ max_usage: 0 });
});
- });
- });
- describe("When instance limit is set to 0", () => {
- beforeEach(() => {
- // Enable Metabot with a configured LLM key
H.updateSetting("metabot-enabled?", true);
- // Set messages as the limit type (easier to trigger with 0)
- // (These settings are added in PR #71699, so we call the API directly)
- cy.request("PUT", "/api/setting/metabot-limit-unit", {
- value: "messages",
- });
-
- // Set a custom quota-reached message
- cy.request("PUT", "/api/setting/metabot-quota-reached-message", {
- value: DEFAULT_QUOTA_MESSAGE,
- });
-
- // Set instance limit to 0 — any usage will immediately exceed it
- cy.request("PUT", "/api/ee/ai-controls/usage/instance", {
- max_usage: 0,
- });
-
- cy.intercept("POST", "/api/metabot/agent-streaming").as("agentReq");
cy.intercept("GET", "/api/automagic-dashboards/database/*/candidates").as(
"xrayCandidates",
);
-
- llmMockServerSetup();
- });
-
- afterEach(() => {
- llmMockServerTeardown();
- });
-
- it("should show the quota-reached message when the user sends a message to Metabot", () => {
cy.visit("/");
cy.wait("@xrayCandidates");
H.openMetabotViaSearchButton();
H.sendMetabotMessage("hello");
- // The backend returns the quota-reached message when limit is exceeded
- H.lastChatMessage().should("have.text", DEFAULT_QUOTA_MESSAGE);
+ H.lastChatMessage().should(
+ "have.text",
+ "You have hit the AI usage limit.",
+ );
});
});
@@ -526,7 +468,7 @@ describe("AI controls > AI usage limits", () => {
});
});
- it("should display both groups with their configured limits and note that users get the highest limit", () => {
+ it("should display both groups with their configured limits and respect the effective user limit (max across their groups)", () => {
cy.visit(AI_USAGE_LIMITS_URL);
// Both groups should appear in the group limits table with correct values
@@ -546,9 +488,7 @@ describe("AI controls > AI usage limits", () => {
.scrollIntoView()
.should("be.visible");
});
- });
- it("should respect the effective user limit (max across their groups)", () => {
// Normal user is in group A (limit 5) and group B (limit 100).
// Effective limit = max(1, 1, 1, 5, 100) = 100.
// (pre-existing groups are set to 1 in beforeEach)
@@ -629,7 +569,6 @@ describe("AI Controls > Tenant usage limits", () => {
});
});
- cy.intercept("POST", "/api/metabot/agent-streaming").as("agentReq");
cy.intercept("GET", "/api/automagic-dashboards/database/*/candidates").as(
"xrayCandidates",
);
@@ -643,7 +582,23 @@ describe("AI Controls > Tenant usage limits", () => {
llmMockServerTeardown();
});
- it("should allow updating tenant limits when tenants are enabled", () => {
+ it("should update tenant limits and apply them to tenant users", () => {
+ cy.log("no tenant limit: the user gets an LLM response");
+ signInAsTenantUser();
+
+ cy.visit("/");
+ cy.wait("@xrayCandidates");
+
+ H.openMetabotViaSearchButton();
+ H.sendMetabotMessage("hello");
+
+ H.lastChatMessage().should("contain.text", MOCK_LLM_RESPONSE);
+ H.lastChatMessage().should("not.contain.text", DEFAULT_QUOTA_MESSAGE);
+
+ cy.log(
+ "tenant limit set to 0 in the UI: the user gets the quota-reached message",
+ );
+ cy.signInAsAdmin();
cy.visit("/admin/metabot/usage-controls/ai-usage-limits");
cy.findByRole("tab", { name: "Specific tenants" }).click();
@@ -652,25 +607,20 @@ describe("AI Controls > Tenant usage limits", () => {
cy.findByTestId("tenant-limits-tab")
.findByText("Test Corp")
.should("be.visible");
- cy.findByLabelText("Max total monthly tokens for Test Corp").type("10");
- cy.wait("@updateTenantLimit").its("response.statusCode").should("eq", 200);
- });
-
- it("should show the quota-reached message when the tenant limit is set to 0 and the user sends a message", () => {
- // Set tenant limit to 0 — any usage will immediately exceed it
- cy.request("PUT", `/api/ee/ai-controls/usage/tenant/${tenantId}`, {
- max_usage: 0,
+ cy.findByLabelText("Max total monthly tokens for Test Corp").type("0");
+ cy.wait("@updateTenantLimit").then(({ request, response }) => {
+ expect(request.url).to.match(
+ new RegExp(`/api/ee/ai-controls/usage/tenant/${tenantId}$`),
+ );
+ expect(request.body).to.deep.equal({ max_usage: 0 });
+ expect(response?.statusCode).to.eq(200);
});
// Clear the backend limit-check cache so the new limit is seen immediately,
cy.request("DELETE", "/api/testing/metabot/seed-ai-usage", {
user_id: tenantUserId,
});
- // Sign in as the tenant user
- cy.request("POST", "/api/session", {
- username: TENANT_USER_EMAIL,
- password: TENANT_USER_PASSWORD,
- });
+ signInAsTenantUser();
cy.visit("/");
cy.wait("@xrayCandidates");
@@ -682,36 +632,17 @@ describe("AI Controls > Tenant usage limits", () => {
H.lastChatMessage().should("contain.text", DEFAULT_QUOTA_MESSAGE);
});
- it("should not show the quota-reached message when no tenant limit is set", () => {
- // No tenant limit is set — the chat input should be available without a quota message
- cy.request("PUT", `/api/ee/ai-controls/usage/tenant/${tenantId}`, {
- max_usage: null,
- });
- // Clear the backend limit-check cache so the new null limit is seen immediately,
- cy.request("DELETE", "/api/testing/metabot/seed-ai-usage", {
- user_id: tenantUserId,
- });
-
- // Sign in as the tenant user
+ function signInAsTenantUser() {
cy.request("POST", "/api/session", {
username: TENANT_USER_EMAIL,
password: TENANT_USER_PASSWORD,
});
-
- cy.visit("/");
- cy.wait("@xrayCandidates");
-
- H.openMetabotViaSearchButton();
- H.sendMetabotMessage("hello");
-
- H.lastChatMessage().should("contain.text", MOCK_LLM_RESPONSE);
- H.lastChatMessage().should("not.contain.text", DEFAULT_QUOTA_MESSAGE);
- });
+ }
});
-// A tiny valid PNG (1×1 transparent pixel) as a data URI, used for icon upload tests
-const TINY_PNG_DATA_URI =
- "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==";
+// A tiny valid PNG (1×1 transparent pixel), used for the icon upload test
+const TINY_PNG_BASE64 =
+ "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==";
const MOCK_LLM_PORT = 6123;
const MOCK_LLM_RESPONSE = "Hello from mock LLM!";
const DEFAULT_QUOTA_MESSAGE =
diff --git a/e2e/test/scenarios/metabot/metabot-query-builder.cy.spec.ts b/e2e/test/scenarios/metabot/metabot-query-builder.cy.spec.ts
index 57364fac97b..4887b7dfd2c 100644
--- a/e2e/test/scenarios/metabot/metabot-query-builder.cy.spec.ts
+++ b/e2e/test/scenarios/metabot/metabot-query-builder.cy.spec.ts
@@ -21,68 +21,32 @@ describe("Metabot Query Builder", () => {
cy.signInAsAdmin();
H.activateToken("pro-self-hosted");
H.setupAnthropicLlmProvider();
- cy.intercept("POST", "/api/metabot/agent-streaming").as("agentReq");
});
- it("should show setup guidance when llm-metabot-configured? is false", () => {
- H.clearLlmProviders();
- cy.visit("/question/ask");
- cy.url().should("include", "/question/ask");
- cy.findByRole("button", { name: "connect to a model" }).should(
- "be.visible",
- );
- cy.findByRole("button", { name: "connect to a model" }).click();
- cy.findByTestId("ai-provider-configuration-modal").should("be.visible");
- });
-
- it("should redirect to notebook when metabot-enabled? is false", () => {
- H.updateSetting("metabot-enabled?", false);
-
- cy.log(
- "visiting '/question/ask' should redirect to notebook when metabot is disabled",
- );
- cy.visit("/question/ask");
- cy.url().should("include", "/question#");
- cy.findByTestId("metabot-chat").should("not.exist");
- });
-
- it("should render the agent's reply inline without leaving the page", () => {
+ it("should render errors and the agent's reply inline and support clicking suggested prompts", () => {
cy.visit("/question/ask");
H.metabotChatInput().should("be.visible");
- H.mockMetabotResponse({
- body: mockTextOnlyResponse("Here's what I found."),
- });
+ H.mockMetabotResponse({ body: mockErrorResponse });
H.sendMetabotMessage("Tell me about my data");
- // the reply renders inline in the full-page conversation...
+ // the full-page conversation uses the nlq profile
cy.wait("@metabotAgent").then(({ request }) => {
- // the full-page conversation uses the nlq profile
expect(request.body.profile_id).to.equal("nlq");
});
- H.lastChatMessage().should("have.text", "Here's what I found.");
+ // the error renders inline in the full-page conversation...
+ H.lastChatMessage().should("contain.text", "Something went wrong");
// ...and we move to the conversation's permalink
cy.url().should("include", "/metabot/conversation/");
- });
-
- it("should render a generated chart inline without leaving the page", () => {
- cy.visit("/question/ask");
- H.metabotChatInput().should("be.visible");
H.mockMetabotResponse({
- body: mockGeneratedEntityResponse(allOrdersQuestion.dataset_query),
+ body: mockTextOnlyResponse("Here's what I found."),
});
- H.sendMetabotMessage("Show me all orders");
-
- cy.wait("@metabotAgent");
- // the chart renders inline rather than in the query builder
- cy.findByTestId("metabot-inline-chart").should("be.visible");
- cy.findByTestId("qb-header").should("not.exist");
- cy.url().should("include", "/metabot/conversation/");
- });
+ H.sendMetabotMessage("Tell me about my data");
+ H.lastChatMessage().should("have.text", "Here's what I found.");
- it("should support clicking suggested prompts", () => {
+ cy.log("suggested prompts");
// mock suggested prompts
cy.intercept("GET", "/api/metabot/metabot/*/prompt-suggestions*", {
prompts: [{ prompt: "Show me all orders" }],
@@ -101,23 +65,9 @@ describe("Metabot Query Builder", () => {
// the chart renders inline rather than in the query builder
cy.wait("@metabotAgent");
cy.findByTestId("metabot-inline-chart").should("be.visible");
+ cy.findByTestId("qb-header").should("not.exist");
cy.url().should("include", "/metabot/conversation/");
});
-
- it("should handle errors", () => {
- // visit AI exploration page
- cy.visit("/question/ask");
- H.metabotChatInput().should("be.visible");
-
- // mock the agent request to stream an error
- H.mockMetabotResponse({ body: mockErrorResponse });
-
- // send a prompt
- H.sendMetabotMessage("Show me all orders");
-
- // should show an error message inline
- H.lastChatMessage().should("contain.text", "Something went wrong");
- });
});
// Response helpers
diff --git a/e2e/test/scenarios/metabot/metabot.cy.spec.ts b/e2e/test/scenarios/metabot/metabot.cy.spec.ts
index d6b3f1df113..b9e2323c188 100644
--- a/e2e/test/scenarios/metabot/metabot.cy.spec.ts
+++ b/e2e/test/scenarios/metabot/metabot.cy.spec.ts
@@ -23,45 +23,22 @@ describe("Metabot UI", () => {
cy.wait("@xrayCandidates");
});
- it("should not show filler element if there are not messages", () => {
+ it("should size the filler element and manage the scroll position as messages are added", () => {
H.openMetabotViaSearchButton();
+ cy.findByTestId("metabot-empty-chat-info").should("be.visible");
H.chatMessages().should("not.exist");
cy.findByTestId("metabot-message-filler").should("not.exist");
- });
-
- it("should correctly size the filler element to take remaining space if messages aren't scrollable", () => {
- H.openMetabotViaSearchButton();
- H.mockMetabotResponse({
- statusCode: 200,
- body: whoIsYourFavoriteResponse,
- });
-
- H.sendMetabotMessage("Who is your favorite?");
- cy.findByTestId("metabot-chat-inner-messages")
- .invoke("innerHeight")
- .then((containerHeight) => {
- cy.findByTestId("metabot-chat-inner-messages")
- .children()
- .then(($children) => {
- // span from the first child to the last so inter-message margins count
- const contentHeight =
- $children[$children.length - 1].getBoundingClientRect().bottom -
- $children[0].getBoundingClientRect().top;
- expect(containerHeight).not.to.equal(undefined);
- // we can get some subpixel differences, this isn't a big deal
- expect(contentHeight).to.be.closeTo(containerHeight ?? 0, 1);
- });
- });
- });
-
- it("should resize filler element and auto-scroll to new prompt on subsequent messages", () => {
- H.openMetabotViaSearchButton();
+ cy.log(
+ "if the messages aren't scrollable, the filler takes the remaining space",
+ );
H.mockMetabotResponse({
statusCode: 200,
body: whoIsYourFavoriteResponse,
});
H.sendMetabotMessage("Who is your favorite?");
+ H.lastChatMessage().should("have.text", "You, but don't tell anyone.");
+ assertFillerReachesContainerBottom();
cy.log("test on message shorter than prompt");
H.mockMetabotResponse({
@@ -69,16 +46,15 @@ describe("Metabot UI", () => {
body: H.createMetabotSSEBody(H.metabotTextPart(loremIpsum.repeat(5))),
});
H.sendMetabotMessage("You really mean that?");
+ H.lastChatMessage().should("contain.text", "Lorem ipsum");
+
cy.log("scroll new prompt to top of the scroll area");
- cy.findByTestId("metabot-chat-inner-messages")
- .findByText("You really mean that?")
- .invoke("scrollTop")
- .then((scrollTop) => expect(scrollTop).to.equal(0));
+ assertLastPromptAtContainerTop();
cy.log(
"if the response is shorter than the scroll area, filler should have height",
);
- cy.findByTestId("metabot-message-filler").then(($el) => {
+ cy.findByTestId("metabot-message-filler").should(($el) => {
expect($el[0].clientHeight).to.be.greaterThan(0);
});
@@ -91,37 +67,47 @@ describe("Metabot UI", () => {
cy.log(
"if the response is longer than the scroll area the filler height should be zero",
);
- cy.findByTestId("metabot-message-filler").then(($el) => {
+ cy.findByTestId("metabot-message-filler").should(($el) => {
expect($el[0].clientHeight).to.equal(0);
});
- });
-
- it("should open metabot to the bottom of the conversation when reopened with message history", () => {
- H.mockMetabotResponse({
- statusCode: 200,
- body: H.createMetabotSSEBody(H.metabotTextPart(loremIpsum.repeat(5))),
- });
- H.openMetabotViaSearchButton();
- H.sendMetabotMessage("Who is your favorite?");
+ assertLastPromptAtContainerTop();
+ cy.log(
+ "open metabot to the bottom of the conversation when reopened with message history",
+ );
H.closeMetabotViaCloseButton();
H.openMetabotViaSearchButton();
- cy.findByTestId("metabot-chat-inner-messages").then(($el) => {
+ cy.findByTestId("metabot-chat-messages").should(($el) => {
const el = $el[0];
- const isAtBottom = el.scrollTop + el.clientHeight >= el.scrollHeight;
- expect(isAtBottom).to.be.true;
+ expect(el.scrollHeight).to.be.greaterThan(el.clientHeight);
+ expect(el.scrollTop + el.clientHeight).to.be.closeTo(
+ el.scrollHeight,
+ 1,
+ );
});
});
});
+ it("should not submit a prompt via /metabot/new and redirect /question/ask to the notebook when metabot is disabled", () => {
+ H.updateSetting("metabot-enabled?", false);
+ cy.visit("/metabot/new?q=Who%20is%20your%20favorite%3F");
+ cy.url().should("eq", Cypress.config().baseUrl + "/");
+ cy.wait("@xrayCandidates");
+ H.assertChatVisibility("not.visible");
+ cy.get("@agentReq.all").should("have.length", 0);
+
+ cy.log(
+ "visiting '/question/ask' should redirect to notebook when metabot is disabled",
+ );
+ cy.visit("/question/ask");
+ cy.url().should("include", "/question#");
+ cy.findByTestId("metabot-chat").should("not.exist");
+ });
+
describe("metabot events", () => {
beforeEach(() => {
H.resetSnowplow();
- H.restore();
- cy.signInAsAdmin();
H.enableTracking();
- H.activateToken("pro-self-hosted");
- H.setupAnthropicLlmProvider();
});
afterEach(() => {
@@ -134,16 +120,7 @@ describe("Metabot UI", () => {
cy.wait("@xrayCandidates");
});
- it("should be able to be opened and closed", () => {
- H.openMetabotViaSearchButton();
- H.expectUnstructuredSnowplowEvent({
- event: "metabot_chat_opened",
- triggered_from: "header",
- });
- H.closeMetabotViaCloseButton();
- });
-
- it("should be controlled via keyboard shortcut", () => {
+ it("should be controlled via keyboard shortcut, be able to be opened and closed, send a message to the agent, handle successful or failed responses, and start a new conversation via /metabot/new", () => {
H.openMetabotViaShortcutKey();
H.expectUnstructuredSnowplowEvent({
event: "metabot_chat_opened",
@@ -158,9 +135,22 @@ describe("Metabot UI", () => {
},
1,
);
- });
- it("should allow a user to send a message to the agent and handle successful or failed responses", () => {
+ H.openMetabotViaSearchButton();
+ H.expectUnstructuredSnowplowEvent({
+ event: "metabot_chat_opened",
+ triggered_from: "header",
+ });
+ H.expectUnstructuredSnowplowEvent(
+ {
+ event: "metabot_chat_opened",
+ triggered_from: "keyboard_shortcut",
+ },
+ 1,
+ );
+ H.closeMetabotViaCloseButton();
+
+ cy.log("send a message and handle successful or failed responses");
H.openMetabotViaSearchButton();
H.chatMessages().should("not.exist");
@@ -178,9 +168,8 @@ describe("Metabot UI", () => {
H.mockMetabotResponse({ statusCode: 200, body: apiKeyInvalidResponse });
H.sendMetabotMessage("Who is your favorite?");
H.lastChatMessage().should("contain.text", "Something went wrong");
- });
- it("should allow starting a new metabot conversation via the /metabot/new", () => {
+ cy.log("start a new conversation via /metabot/new");
H.mockMetabotResponse({
statusCode: 200,
body: whoIsYourFavoriteResponse,
@@ -189,14 +178,6 @@ describe("Metabot UI", () => {
H.assertChatVisibility("visible");
H.lastChatMessage().should("have.text", "You, but don't tell anyone.");
});
-
- it("should not submit a prompt via /metabot/new when metabot is disabled", () => {
- H.updateSetting("metabot-enabled?", false);
- cy.visit("/metabot/new?q=Who%20is%20your%20favorite%3F");
- cy.url().should("eq", Cypress.config().baseUrl + "/");
- H.assertChatVisibility("not.visible");
- cy.get("@agentReq.all").should("have.length", 0);
- });
});
});
});
@@ -209,7 +190,11 @@ describe("Metabot in full-app embedding", () => {
H.setupAnthropicLlmProvider();
});
- it("should show the metabot button when embedded-metabot-enabled? is true", () => {
+ it("should show the metabot button only when embedded-metabot-enabled? is true", () => {
+ cy.intercept("GET", "/api/metabot/permissions/user-permissions").as(
+ "getUserPermissions",
+ );
+
H.updateEnterpriseSettings({ "embedded-metabot-enabled?": true });
H.visitFullAppEmbeddingUrl({
@@ -217,11 +202,13 @@ describe("Metabot in full-app embedding", () => {
qs: {},
});
+ cy.wait("@getUserPermissions");
H.appBar().icon("metabot").should("be.visible");
- });
- it("should not show the metabot button when embedded-metabot-enabled? is false", () => {
H.updateEnterpriseSettings({ "embedded-metabot-enabled?": false });
+ cy.intercept("GET", "/api/metabot/permissions/user-permissions").as(
+ "getUserPermissionsWhenDisabled",
+ );
H.visitFullAppEmbeddingUrl({
url: `/question/${ORDERS_BY_YEAR_QUESTION_ID}`,
@@ -233,6 +220,7 @@ describe("Metabot in full-app embedding", () => {
cy.log("Assert metabot buttons are not rendered");
H.appBar().icon("metabot").should("not.exist");
+ cy.get("@getUserPermissionsWhenDisabled.all").should("have.length", 0);
});
});
@@ -248,3 +236,38 @@ const apiKeyInvalidResponse = H.createMetabotSSEBody(
H.metabotErrorPart("Anthropic API key expired or invalid"),
H.metabotFinishPart("error"),
);
+
+function getContainerPadding($container: JQuery<HTMLElement>) {
+ const style = getComputedStyle($container[0]);
+ return {
+ top: parseFloat(style.paddingTop),
+ bottom: parseFloat(style.paddingBottom),
+ };
+}
+
+function assertFillerReachesContainerBottom() {
+ cy.findByTestId("metabot-chat-messages").should(($container) => {
+ const containerRect = $container[0].getBoundingClientRect();
+ const fillerRect = $container
+ .find("[data-testid='metabot-message-filler']")[0]
+ .getBoundingClientRect();
+ expect(fillerRect.height).to.be.greaterThan(0);
+ expect(fillerRect.bottom).to.be.closeTo(
+ containerRect.bottom - getContainerPadding($container).bottom,
+ 1,
+ );
+ });
+}
+
+function assertLastPromptAtContainerTop() {
+ cy.findByTestId("metabot-chat-messages").should(($container) => {
+ const containerRect = $container[0].getBoundingClientRect();
+ const prompts = $container.find("[data-message-role='user']").toArray();
+ expect(prompts).to.have.length.greaterThan(0);
+ const promptRect = prompts[prompts.length - 1].getBoundingClientRect();
+ expect(promptRect.top).to.be.closeTo(
+ containerRect.top + getContainerPadding($container).top,
+ 2,
+ );
+ });
+}
diff --git a/e2e/test/scenarios/metabot/native-sql-generation.cy.spec.ts b/e2e/test/scenarios/metabot/native-sql-generation.cy.spec.ts
index d2d722640ec..82ff7f402e4 100644
--- a/e2e/test/scenarios/metabot/native-sql-generation.cy.spec.ts
+++ b/e2e/test/scenarios/metabot/native-sql-generation.cy.spec.ts
@@ -17,7 +17,7 @@ const rejectButton = () => cy.findByTestId("reject-proposed-changes-button");
const generatingLoader = () => cy.findByLabelText("Stop generating");
describe("Native SQL generation", () => {
- it("should show setup guidance when metabot is not configured", () => {
+ it("should show setup guidance in SQL generation and on the AI exploration page when metabot is not configured", () => {
H.restore();
cy.signInAsAdmin();
H.activateToken("pro-self-hosted");
@@ -33,6 +33,14 @@ describe("Native SQL generation", () => {
cy.findByRole("button", { name: "connect to a model" }).click();
cy.findByTestId("ai-provider-configuration-modal").should("be.visible");
+
+ cy.visit("/question/ask");
+ cy.url().should("include", "/question/ask");
+ cy.findByRole("button", { name: "connect to a model" }).should(
+ "be.visible",
+ );
+ cy.findByRole("button", { name: "connect to a model" }).click();
+ cy.findByTestId("ai-provider-configuration-modal").should("be.visible");
});
describe("single db", () => {
@@ -41,10 +49,9 @@ describe("Native SQL generation", () => {
cy.signInAsAdmin();
H.activateToken("pro-self-hosted");
H.setupAnthropicLlmProvider({ apiKey: "sk-ant-api03-test-token" });
- cy.intercept("POST", "/api/metabot/agent-streaming").as("agentReq");
});
- it("should be able to successfully generate sql", () => {
+ it("should be able to successfully generate sql and correctly control the input", () => {
H.startNewNativeQuestion({ query: "SELECT 1" });
H.NativeEditor.get().should("be.visible");
@@ -85,9 +92,9 @@ describe("Native SQL generation", () => {
rejectButton().should("not.exist");
H.NativeEditor.get().should("not.contain", "SELECT 1");
H.NativeEditor.get().should("contain", "SELECT * FROM users");
- });
- it("should be able to correctly control the input", () => {
+ cy.log("control the input");
+ cy.visit("/");
H.startNewNativeQuestion();
H.NativeEditor.get().should("be.visible");
@@ -127,10 +134,31 @@ describe("Native SQL generation", () => {
cy.signInAsAdmin();
H.activateToken("pro-self-hosted");
H.setupAnthropicLlmProvider({ apiKey: "sk-ant-api03-test-token" });
- cy.intercept("POST", "/api/metabot/agent-streaming").as("agentReq");
});
- it("should manage conversation state correctly", () => {
+ it("should show error if no code_edit is received and manage conversation state correctly", () => {
+ H.startNewNativeQuestion();
+ H.NativeEditor.get().should("be.visible");
+
+ toggleInlineSQLPrompt();
+ inlinePromptInput().click();
+ cy.realType("do something", { pressDelay: 10 });
+ H.mockMetabotResponse({
+ body: mockTextOnlyResponse("I can help with that!"),
+ });
+ generateButton().click();
+ cy.wait("@metabotAgent");
+
+ errorMessage()
+ .should("be.visible")
+ .and(
+ "have.text",
+ "Sorry, I ran into an error. Could you please try that again?",
+ );
+ acceptButton().should("not.exist");
+
+ cy.log("manage conversation state");
+ cy.visit("/");
H.startNewNativeQuestion({ query: "SELECT 1" });
H.NativeEditor.get().should("be.visible");
@@ -193,8 +221,10 @@ describe("Native SQL generation", () => {
// leave the page, go to new SQL page and send a new prompt
acceptButton().click();
- cy.visit("/");
- H.startNewNativeQuestion();
+ cy.findByTestId("main-logo-link").click();
+ H.modal().button("Discard changes").click();
+ cy.location("pathname").should("eq", "/");
+ H.newButton("SQL query").click();
H.NativeEditor.get().should("be.visible");
toggleInlineSQLPrompt();
inlinePromptInput().click();
@@ -215,23 +245,6 @@ describe("Native SQL generation", () => {
acceptButton().should("not.exist");
rejectButton().should("not.exist");
});
-
- it("should show error if no code_edit is received", () => {
- H.startNewNativeQuestion();
- H.NativeEditor.get().should("be.visible");
-
- toggleInlineSQLPrompt();
- inlinePromptInput().click();
- cy.realType("do something", { pressDelay: 10 });
- H.mockMetabotResponse({
- body: mockTextOnlyResponse("I can help with that!"),
- });
- generateButton().click();
- cy.wait("@metabotAgent");
-
- errorMessage().should("be.visible");
- acceptButton().should("not.exist");
- });
});
});