The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Consolidate Metabot e2e tests (#83567)
Chris Vandoodewaard70d5cd55e734
diff --git a/e2e/support/helpers/e2e-metabot-helpers.ts b/e2e/support/helpers/e2e-metabot-helpers.tsindex 1501412e986..5cbd492efd6 100644--- a/e2e/support/helpers/e2e-metabot-helpers.ts+++ b/e2e/support/helpers/e2e-metabot-helpers.ts@@ -42,10 +42,6 @@ export function setupAnthropicLlmProvider({]);}-export function clearLlmProviders() {- return setLlmProviders([]);-}-export function metabotChatSidebar() {return cy.findByTestId("metabot-chat");}diff --git a/e2e/test/scenarios/metabot/ai-controls.cy.spec.ts b/e2e/test/scenarios/metabot/ai-controls.cy.spec.tsindex 02c365dc489..f9023004470 100644--- a/e2e/test/scenarios/metabot/ai-controls.cy.spec.ts+++ b/e2e/test/scenarios/metabot/ai-controls.cy.spec.ts@@ -1,7 +1,4 @@-import {- ALL_USERS_GROUP_ID,- NORMAL_USER_ID,-} from "e2e/support/cypress_sample_instance_data";+import { NORMAL_USER_ID } from "e2e/support/cypress_sample_instance_data";const { H } = cy;@@ -19,7 +16,7 @@ describe("AI Controls > Metabot access and customization", () => {});describe("Feature Access page", () => {- it("should display the AI feature access table with groups and save a permission change", () => {+ it("should display the AI feature access table with groups, save a permission change, and hide the Metabot chat icon for users in a group without Metabot access", () => {cy.intercept("GET", "/api/ee/ai-controls/permissions").as("getPermissions",);@@ -61,9 +58,34 @@ describe("AI Controls > Metabot access and customization", () => {cy.findAllByRole("checkbox").each(($checkbox) => {cy.wrap($checkbox).should("not.be.checked");});+ });++ cy.intercept("GET", "/api/metabot/permissions/user-permissions").as(+ "getUserPermissions",+ );++ // Sign in as a normal user (who is only in the All Users group)+ cy.signInAsNormalUser();+ cy.visit("/");+ cy.wait("@getUserPermissions")+ .its("response.body.permissions.metabot")+ .should("eq", "no");++ // Wait for the navigation bar to be present+ cy.findByLabelText("Navigation bar").should("be.visible");++ // The Metabot chat icon should not be present+ H.appBar()+ .findByRole("button", { name: /Chat with/ })+ .should("not.exist");+ cy.signInAsAdmin();+ cy.visit("/admin/metabot/usage-controls/ai-feature-access");+ cy.wait("@getPermissions");++ cy.findByRole("row", { name: /All Users permissions/ }).within(() => {// Toggle the metabot AI features switch on again- cy.findByRole("switch").click({ force: true });+ cy.findByRole("switch").should("not.be.checked").click({ force: true });cy.wait("@updatePermissions").its("response.statusCode")@@ -74,50 +96,35 @@ describe("AI Controls > Metabot access and customization", () => {cy.wrap($checkbox).should("be.checked");});});- });- });-- describe("Group access controls", () => {- it("should not show the Metabot chat icon for users in a group without Metabot access", () => {- // First, get the current permissions so we can update only the All Users group- cy.request("GET", "/api/ee/ai-controls/permissions").then((response) => {- const currentPermissions: Array<{- group_id: number;- perm_type: string;- perm_value: string;- }> = response.body.permissions;-- // Set All Users group's metabot permission to "no"- const updatedPermissions = currentPermissions.map((p) => {- if (- p.group_id === ALL_USERS_GROUP_ID &&- p.perm_type === "permission/metabot"- ) {- return { ...p, perm_value: "no" };- }- return p;- });- cy.request("PUT", "/api/ee/ai-controls/permissions", {- permissions: updatedPermissions,- });- });-- // Sign in as a normal user (who is only in the All Users group)+ cy.log(+ "a user in a group with Metabot access sees the Metabot chat icon",+ );+ cy.intercept("GET", "/api/metabot/permissions/user-permissions").as(+ "getNormalUserPermissions",+ );cy.signInAsNormalUser();cy.visit("/");-- // Wait for the navigation bar to be present- cy.findByLabelText("Navigation bar").should("be.visible");-- // The Metabot chat icon should not be present- H.appBar().find('[aria-label*="Chat with"]').should("not.exist");+ cy.wait("@getNormalUserPermissions")+ .its("response.body.permissions.metabot")+ .should("eq", "yes");+ H.appBar()+ .findByRole("button", { name: /Chat with/ })+ .should("be.visible");});});describe("Customization", () => {- it("should save a custom Metabot name", () => {+ it("should save a custom Metabot name, icon and illustrations setting and use them in the app", () => {+ H.updateEnterpriseSettings({+ "metabot-show-illustrations": true,+ });+cy.intercept("PUT", "/api/setting/metabot-name").as("saveName");+ cy.intercept("PUT", "/api/setting/metabot-icon").as("saveIcon");+ cy.intercept("PUT", "/api/setting/metabot-show-illustrations").as(+ "saveIllustrations",+ );cy.visit("/admin/metabot/customization");@@ -132,16 +139,6 @@ describe("AI Controls > Metabot access and customization", () => {// Wait for debounced savecy.wait("@saveName").its("response.statusCode").should("eq", 204);- // Reload and verify persistence- cy.reload();- cy.findByLabelText("AI agent's name").should("have.value", "HAL 9000");- });-- it("should upload a custom Metabot icon", () => {- cy.intercept("PUT", "/api/setting/metabot-icon").as("saveIcon");-- cy.visit("/admin/metabot/customization");-H.main().findByText("AI agent's icon").should("be.visible");cy.findByRole("button", { name: "Upload a custom icon" }).should("be.visible",@@ -150,10 +147,7 @@ describe("AI Controls > Metabot access and customization", () => {// Upload a tiny PNG via the hidden file inputcy.get('input[type="file"]').selectFile({- contents: Cypress.Buffer.from(- "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==",- "base64",- ),+ contents: Cypress.Buffer.from(TINY_PNG_BASE64, "base64"),fileName: "metabot-icon.png",mimeType: "image/png",},@@ -164,19 +158,6 @@ describe("AI Controls > Metabot access and customization", () => {// The "Remove custom icon" button should be visiblecy.findByLabelText("Remove custom icon").should("be.visible");- });-- it("should hide Metabot illustrations when the toggle is switched off", () => {- H.updateEnterpriseSettings({- "metabot-show-illustrations": true,- });- H.updateSetting("metabot-enabled?", true);-- cy.intercept("PUT", "/api/setting/metabot-show-illustrations").as(- "saveIllustrations",- );-- cy.visit("/admin/metabot/customization");H.main().findByText("Metabot illustrations").should("be.visible");@@ -196,9 +177,23 @@ describe("AI Controls > Metabot access and customization", () => {.its("response.statusCode").should("eq", 204);- // Navigate to the home page and open Metabot chat to verify illustrations are hiddency.visit("/");- H.openMetabotViaSearchButton();+ cy.findByLabelText("Navigation bar").should("be.visible");++ // The app bar button tooltip/aria-label should reflect the custom name+ H.appBar()+ .findByRole("button", { name: /Chat with HAL 9000/ })+ .should("be.visible")+ .within(() => {+ // When a custom icon is set, MetabotIcon renders an <img> with alt = metabotName+ cy.get("img").should("be.visible");+ });++ // Open Metabot chat to verify illustrations are hidden+ H.appBar()+ .findByRole("button", { name: /Chat with HAL 9000/ })+ .click();+ H.assertChatVisibility("visible");cy.findByTestId("metabot-empty-chat-info").should("be.visible");// The SVG illustration should NOT be rendered when showIllustrations=false@@ -210,40 +205,16 @@ describe("AI Controls > Metabot access and customization", () => {.findByText(/Explore your metrics and models with AI/).should("be.visible");});-- it("should show the custom Metabot name in the app bar button tooltip", () => {- H.updateSetting("metabot-name", "Aria");-- cy.visit("/");- cy.findByLabelText("Navigation bar").should("be.visible");-- // The app bar button tooltip/aria-label should reflect the custom name- H.appBar()- .findByRole("button", { name: /Chat with Aria/ })- .should("be.visible");- });-- it("should show a custom Metabot icon in the app bar when metabot-icon is set", () => {- H.updateEnterpriseSettings({ "metabot-icon": TINY_PNG_DATA_URI });-- cy.visit("/");- cy.findByLabelText("Navigation bar").should("be.visible");-- // The custom icon img should be rendered (alt = metabot name)- H.appBar()- .findByRole("button", { name: /Chat with/ })- .within(() => {- // When a custom icon is set, MetabotIcon renders an <img> with alt = metabotName- cy.get("img").should("be.visible");- });- });});describe("System Prompts pages", () => {- it("should save a custom Metabot chat system prompt", () => {+ it("should save custom Metabot chat and SQL generation system prompts", () => {cy.intercept("PUT", "/api/setting/metabot-chat-system-prompt").as("savePrompt",);+ cy.intercept("PUT", "/api/setting/metabot-sql-system-prompt").as(+ "saveSqlPrompt",+ );cy.visit("/admin/metabot/system-prompts/metabot-chat");@@ -260,19 +231,9 @@ describe("AI Controls > Metabot access and customization", () => {cy.wait("@savePrompt").its("response.statusCode").should("eq", 204);- // Reload and verify persistence- cy.reload();- cy.findByRole("textbox", {- name: /AI chat prompt instructions/,- }).should("have.text", "Be concise and helpful.");- });-- it("should save a custom SQL generation system prompt", () => {- cy.intercept("PUT", "/api/setting/metabot-sql-system-prompt").as(- "saveSqlPrompt",- );-- cy.visit("/admin/metabot/system-prompts/sql-generation");+ cy.findByTestId("admin-layout-sidebar")+ .findByRole("link", { name: "SQL generation" })+ .click();cy.findByRole("heading", {name: "SQL generation prompt instructions",@@ -289,10 +250,18 @@ describe("AI Controls > Metabot access and customization", () => {cy.wait("@saveSqlPrompt").its("response.statusCode").should("eq", 204);+ // Reload and verify persistencecy.reload();cy.findByRole("textbox", {name: /SQL generation prompt instructions/,}).should("have.text", "Always use uppercase SQL keywords.");++ cy.findByTestId("admin-layout-sidebar")+ .findByRole("link", { name: "AI chat" })+ .click();+ cy.findByRole("textbox", {+ name: /AI chat prompt instructions/,+ }).should("have.text", "Be concise and helpful.");});});});@@ -311,7 +280,7 @@ describe("AI controls > AI usage limits", () => {llmMockServerTeardown();});- describe("AI Limits settings can be saved properly", () => {+ describe("Instance limit settings and enforcement", () => {beforeEach(() => {cy.intercept("GET", "/api/ee/ai-controls/usage/instance").as("getInstanceLimit",@@ -333,10 +302,22 @@ describe("AI controls > AI usage limits", () => {);});- it("should save limit type, reset period, instance limit, and quota-reached message when changed", () => {+ it("should clear and save instance limit settings, then enforce a zero limit in chat", () => {+ // Pre-set a limit so there's something to clear+ cy.request("PUT", "/api/ee/ai-controls/usage/instance", {+ max_usage: 100,+ });+cy.visit(AI_USAGE_LIMITS_URL);cy.wait("@getInstanceLimit");+ cy.findByRole("textbox", {+ name: "Total monthly instance token limit",+ }).clear();+ cy.wait("@updateInstanceLimit").then(({ request }) => {+ expect(request.body).to.deep.equal({ max_usage: null });+ });+// Change limit type to messagescy.findByRole("radio", { name: "By message count" }).click({force: true,@@ -366,68 +347,29 @@ describe("AI controls > AI usage limits", () => {value: "You have hit the AI usage limit.",});});- });-- it("should save instance limit as null when the field is cleared", () => {- // Pre-set a limit so there's something to clear- cy.request("PUT", "/api/ee/ai-controls/usage/instance", {- max_usage: 100,- });-- cy.visit(AI_USAGE_LIMITS_URL);- cy.wait("@getInstanceLimit");-- cy.findByRole("textbox", {- name: "Total monthly instance token limit",- }).clear();+ cy.log("Enforce a zero instance limit in a fresh chat");+ cy.findByLabelText("Total weekly instance message limit").type(+ "{selectall}0",+ );cy.wait("@updateInstanceLimit").then(({ request }) => {- expect(request.body).to.deep.equal({ max_usage: null });+ expect(request.body).to.deep.equal({ max_usage: 0 });});- });- });- describe("When instance limit is set to 0", () => {- beforeEach(() => {- // Enable Metabot with a configured LLM keyH.updateSetting("metabot-enabled?", true);- // Set messages as the limit type (easier to trigger with 0)- // (These settings are added in PR #71699, so we call the API directly)- cy.request("PUT", "/api/setting/metabot-limit-unit", {- value: "messages",- });-- // Set a custom quota-reached message- cy.request("PUT", "/api/setting/metabot-quota-reached-message", {- value: DEFAULT_QUOTA_MESSAGE,- });-- // Set instance limit to 0 — any usage will immediately exceed it- cy.request("PUT", "/api/ee/ai-controls/usage/instance", {- max_usage: 0,- });-- cy.intercept("POST", "/api/metabot/agent-streaming").as("agentReq");cy.intercept("GET", "/api/automagic-dashboards/database/*/candidates").as("xrayCandidates",);-- llmMockServerSetup();- });-- afterEach(() => {- llmMockServerTeardown();- });-- it("should show the quota-reached message when the user sends a message to Metabot", () => {cy.visit("/");cy.wait("@xrayCandidates");H.openMetabotViaSearchButton();H.sendMetabotMessage("hello");- // The backend returns the quota-reached message when limit is exceeded- H.lastChatMessage().should("have.text", DEFAULT_QUOTA_MESSAGE);+ H.lastChatMessage().should(+ "have.text",+ "You have hit the AI usage limit.",+ );});});@@ -526,7 +468,7 @@ describe("AI controls > AI usage limits", () => {});});- it("should display both groups with their configured limits and note that users get the highest limit", () => {+ it("should display both groups with their configured limits and respect the effective user limit (max across their groups)", () => {cy.visit(AI_USAGE_LIMITS_URL);// Both groups should appear in the group limits table with correct values@@ -546,9 +488,7 @@ describe("AI controls > AI usage limits", () => {.scrollIntoView().should("be.visible");});- });- it("should respect the effective user limit (max across their groups)", () => {// Normal user is in group A (limit 5) and group B (limit 100).// Effective limit = max(1, 1, 1, 5, 100) = 100.// (pre-existing groups are set to 1 in beforeEach)@@ -629,7 +569,6 @@ describe("AI Controls > Tenant usage limits", () => {});});- cy.intercept("POST", "/api/metabot/agent-streaming").as("agentReq");cy.intercept("GET", "/api/automagic-dashboards/database/*/candidates").as("xrayCandidates",);@@ -643,7 +582,23 @@ describe("AI Controls > Tenant usage limits", () => {llmMockServerTeardown();});- it("should allow updating tenant limits when tenants are enabled", () => {+ it("should update tenant limits and apply them to tenant users", () => {+ cy.log("no tenant limit: the user gets an LLM response");+ signInAsTenantUser();++ cy.visit("/");+ cy.wait("@xrayCandidates");++ H.openMetabotViaSearchButton();+ H.sendMetabotMessage("hello");++ H.lastChatMessage().should("contain.text", MOCK_LLM_RESPONSE);+ H.lastChatMessage().should("not.contain.text", DEFAULT_QUOTA_MESSAGE);++ cy.log(+ "tenant limit set to 0 in the UI: the user gets the quota-reached message",+ );+ cy.signInAsAdmin();cy.visit("/admin/metabot/usage-controls/ai-usage-limits");cy.findByRole("tab", { name: "Specific tenants" }).click();@@ -652,25 +607,20 @@ describe("AI Controls > Tenant usage limits", () => {cy.findByTestId("tenant-limits-tab").findByText("Test Corp").should("be.visible");- cy.findByLabelText("Max total monthly tokens for Test Corp").type("10");- cy.wait("@updateTenantLimit").its("response.statusCode").should("eq", 200);- });-- it("should show the quota-reached message when the tenant limit is set to 0 and the user sends a message", () => {- // Set tenant limit to 0 — any usage will immediately exceed it- cy.request("PUT", `/api/ee/ai-controls/usage/tenant/${tenantId}`, {- max_usage: 0,+ cy.findByLabelText("Max total monthly tokens for Test Corp").type("0");+ cy.wait("@updateTenantLimit").then(({ request, response }) => {+ expect(request.url).to.match(+ new RegExp(`/api/ee/ai-controls/usage/tenant/${tenantId}$`),+ );+ expect(request.body).to.deep.equal({ max_usage: 0 });+ expect(response?.statusCode).to.eq(200);});// Clear the backend limit-check cache so the new limit is seen immediately,cy.request("DELETE", "/api/testing/metabot/seed-ai-usage", {user_id: tenantUserId,});- // Sign in as the tenant user- cy.request("POST", "/api/session", {- username: TENANT_USER_EMAIL,- password: TENANT_USER_PASSWORD,- });+ signInAsTenantUser();cy.visit("/");cy.wait("@xrayCandidates");@@ -682,36 +632,17 @@ describe("AI Controls > Tenant usage limits", () => {H.lastChatMessage().should("contain.text", DEFAULT_QUOTA_MESSAGE);});- it("should not show the quota-reached message when no tenant limit is set", () => {- // No tenant limit is set — the chat input should be available without a quota message- cy.request("PUT", `/api/ee/ai-controls/usage/tenant/${tenantId}`, {- max_usage: null,- });- // Clear the backend limit-check cache so the new null limit is seen immediately,- cy.request("DELETE", "/api/testing/metabot/seed-ai-usage", {- user_id: tenantUserId,- });-- // Sign in as the tenant user+ function signInAsTenantUser() {cy.request("POST", "/api/session", {username: TENANT_USER_EMAIL,password: TENANT_USER_PASSWORD,});-- cy.visit("/");- cy.wait("@xrayCandidates");-- H.openMetabotViaSearchButton();- H.sendMetabotMessage("hello");-- H.lastChatMessage().should("contain.text", MOCK_LLM_RESPONSE);- H.lastChatMessage().should("not.contain.text", DEFAULT_QUOTA_MESSAGE);- });+ }});-// A tiny valid PNG (1×1 transparent pixel) as a data URI, used for icon upload tests-const TINY_PNG_DATA_URI =- "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==";+// A tiny valid PNG (1×1 transparent pixel), used for the icon upload test+const TINY_PNG_BASE64 =+ "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==";const MOCK_LLM_PORT = 6123;const MOCK_LLM_RESPONSE = "Hello from mock LLM!";const DEFAULT_QUOTA_MESSAGE =diff --git a/e2e/test/scenarios/metabot/metabot-query-builder.cy.spec.ts b/e2e/test/scenarios/metabot/metabot-query-builder.cy.spec.tsindex 57364fac97b..4887b7dfd2c 100644--- a/e2e/test/scenarios/metabot/metabot-query-builder.cy.spec.ts+++ b/e2e/test/scenarios/metabot/metabot-query-builder.cy.spec.ts@@ -21,68 +21,32 @@ describe("Metabot Query Builder", () => {cy.signInAsAdmin();H.activateToken("pro-self-hosted");H.setupAnthropicLlmProvider();- cy.intercept("POST", "/api/metabot/agent-streaming").as("agentReq");});- it("should show setup guidance when llm-metabot-configured? is false", () => {- H.clearLlmProviders();- cy.visit("/question/ask");- cy.url().should("include", "/question/ask");- cy.findByRole("button", { name: "connect to a model" }).should(- "be.visible",- );- cy.findByRole("button", { name: "connect to a model" }).click();- cy.findByTestId("ai-provider-configuration-modal").should("be.visible");- });-- it("should redirect to notebook when metabot-enabled? is false", () => {- H.updateSetting("metabot-enabled?", false);-- cy.log(- "visiting '/question/ask' should redirect to notebook when metabot is disabled",- );- cy.visit("/question/ask");- cy.url().should("include", "/question#");- cy.findByTestId("metabot-chat").should("not.exist");- });-- it("should render the agent's reply inline without leaving the page", () => {+ it("should render errors and the agent's reply inline and support clicking suggested prompts", () => {cy.visit("/question/ask");H.metabotChatInput().should("be.visible");- H.mockMetabotResponse({- body: mockTextOnlyResponse("Here's what I found."),- });+ H.mockMetabotResponse({ body: mockErrorResponse });H.sendMetabotMessage("Tell me about my data");- // the reply renders inline in the full-page conversation...+ // the full-page conversation uses the nlq profilecy.wait("@metabotAgent").then(({ request }) => {- // the full-page conversation uses the nlq profileexpect(request.body.profile_id).to.equal("nlq");});- H.lastChatMessage().should("have.text", "Here's what I found.");+ // the error renders inline in the full-page conversation...+ H.lastChatMessage().should("contain.text", "Something went wrong");// ...and we move to the conversation's permalinkcy.url().should("include", "/metabot/conversation/");- });-- it("should render a generated chart inline without leaving the page", () => {- cy.visit("/question/ask");- H.metabotChatInput().should("be.visible");H.mockMetabotResponse({- body: mockGeneratedEntityResponse(allOrdersQuestion.dataset_query),+ body: mockTextOnlyResponse("Here's what I found."),});- H.sendMetabotMessage("Show me all orders");-- cy.wait("@metabotAgent");- // the chart renders inline rather than in the query builder- cy.findByTestId("metabot-inline-chart").should("be.visible");- cy.findByTestId("qb-header").should("not.exist");- cy.url().should("include", "/metabot/conversation/");- });+ H.sendMetabotMessage("Tell me about my data");+ H.lastChatMessage().should("have.text", "Here's what I found.");- it("should support clicking suggested prompts", () => {+ cy.log("suggested prompts");// mock suggested promptscy.intercept("GET", "/api/metabot/metabot/*/prompt-suggestions*", {prompts: [{ prompt: "Show me all orders" }],@@ -101,23 +65,9 @@ describe("Metabot Query Builder", () => {// the chart renders inline rather than in the query buildercy.wait("@metabotAgent");cy.findByTestId("metabot-inline-chart").should("be.visible");+ cy.findByTestId("qb-header").should("not.exist");cy.url().should("include", "/metabot/conversation/");});-- it("should handle errors", () => {- // visit AI exploration page- cy.visit("/question/ask");- H.metabotChatInput().should("be.visible");-- // mock the agent request to stream an error- H.mockMetabotResponse({ body: mockErrorResponse });-- // send a prompt- H.sendMetabotMessage("Show me all orders");-- // should show an error message inline- H.lastChatMessage().should("contain.text", "Something went wrong");- });});// Response helpersdiff --git a/e2e/test/scenarios/metabot/metabot.cy.spec.ts b/e2e/test/scenarios/metabot/metabot.cy.spec.tsindex d6b3f1df113..b9e2323c188 100644--- a/e2e/test/scenarios/metabot/metabot.cy.spec.ts+++ b/e2e/test/scenarios/metabot/metabot.cy.spec.ts@@ -23,45 +23,22 @@ describe("Metabot UI", () => {cy.wait("@xrayCandidates");});- it("should not show filler element if there are not messages", () => {+ it("should size the filler element and manage the scroll position as messages are added", () => {H.openMetabotViaSearchButton();+ cy.findByTestId("metabot-empty-chat-info").should("be.visible");H.chatMessages().should("not.exist");cy.findByTestId("metabot-message-filler").should("not.exist");- });-- it("should correctly size the filler element to take remaining space if messages aren't scrollable", () => {- H.openMetabotViaSearchButton();- H.mockMetabotResponse({- statusCode: 200,- body: whoIsYourFavoriteResponse,- });-- H.sendMetabotMessage("Who is your favorite?");- cy.findByTestId("metabot-chat-inner-messages")- .invoke("innerHeight")- .then((containerHeight) => {- cy.findByTestId("metabot-chat-inner-messages")- .children()- .then(($children) => {- // span from the first child to the last so inter-message margins count- const contentHeight =- $children[$children.length - 1].getBoundingClientRect().bottom -- $children[0].getBoundingClientRect().top;- expect(containerHeight).not.to.equal(undefined);- // we can get some subpixel differences, this isn't a big deal- expect(contentHeight).to.be.closeTo(containerHeight ?? 0, 1);- });- });- });-- it("should resize filler element and auto-scroll to new prompt on subsequent messages", () => {- H.openMetabotViaSearchButton();+ cy.log(+ "if the messages aren't scrollable, the filler takes the remaining space",+ );H.mockMetabotResponse({statusCode: 200,body: whoIsYourFavoriteResponse,});H.sendMetabotMessage("Who is your favorite?");+ H.lastChatMessage().should("have.text", "You, but don't tell anyone.");+ assertFillerReachesContainerBottom();cy.log("test on message shorter than prompt");H.mockMetabotResponse({@@ -69,16 +46,15 @@ describe("Metabot UI", () => {body: H.createMetabotSSEBody(H.metabotTextPart(loremIpsum.repeat(5))),});H.sendMetabotMessage("You really mean that?");+ H.lastChatMessage().should("contain.text", "Lorem ipsum");+cy.log("scroll new prompt to top of the scroll area");- cy.findByTestId("metabot-chat-inner-messages")- .findByText("You really mean that?")- .invoke("scrollTop")- .then((scrollTop) => expect(scrollTop).to.equal(0));+ assertLastPromptAtContainerTop();cy.log("if the response is shorter than the scroll area, filler should have height",);- cy.findByTestId("metabot-message-filler").then(($el) => {+ cy.findByTestId("metabot-message-filler").should(($el) => {expect($el[0].clientHeight).to.be.greaterThan(0);});@@ -91,37 +67,47 @@ describe("Metabot UI", () => {cy.log("if the response is longer than the scroll area the filler height should be zero",);- cy.findByTestId("metabot-message-filler").then(($el) => {+ cy.findByTestId("metabot-message-filler").should(($el) => {expect($el[0].clientHeight).to.equal(0);});- });-- it("should open metabot to the bottom of the conversation when reopened with message history", () => {- H.mockMetabotResponse({- statusCode: 200,- body: H.createMetabotSSEBody(H.metabotTextPart(loremIpsum.repeat(5))),- });- H.openMetabotViaSearchButton();- H.sendMetabotMessage("Who is your favorite?");+ assertLastPromptAtContainerTop();+ cy.log(+ "open metabot to the bottom of the conversation when reopened with message history",+ );H.closeMetabotViaCloseButton();H.openMetabotViaSearchButton();- cy.findByTestId("metabot-chat-inner-messages").then(($el) => {+ cy.findByTestId("metabot-chat-messages").should(($el) => {const el = $el[0];- const isAtBottom = el.scrollTop + el.clientHeight >= el.scrollHeight;- expect(isAtBottom).to.be.true;+ expect(el.scrollHeight).to.be.greaterThan(el.clientHeight);+ expect(el.scrollTop + el.clientHeight).to.be.closeTo(+ el.scrollHeight,+ 1,+ );});});});+ it("should not submit a prompt via /metabot/new and redirect /question/ask to the notebook when metabot is disabled", () => {+ H.updateSetting("metabot-enabled?", false);+ cy.visit("/metabot/new?q=Who%20is%20your%20favorite%3F");+ cy.url().should("eq", Cypress.config().baseUrl + "/");+ cy.wait("@xrayCandidates");+ H.assertChatVisibility("not.visible");+ cy.get("@agentReq.all").should("have.length", 0);++ cy.log(+ "visiting '/question/ask' should redirect to notebook when metabot is disabled",+ );+ cy.visit("/question/ask");+ cy.url().should("include", "/question#");+ cy.findByTestId("metabot-chat").should("not.exist");+ });+describe("metabot events", () => {beforeEach(() => {H.resetSnowplow();- H.restore();- cy.signInAsAdmin();H.enableTracking();- H.activateToken("pro-self-hosted");- H.setupAnthropicLlmProvider();});afterEach(() => {@@ -134,16 +120,7 @@ describe("Metabot UI", () => {cy.wait("@xrayCandidates");});- it("should be able to be opened and closed", () => {- H.openMetabotViaSearchButton();- H.expectUnstructuredSnowplowEvent({- event: "metabot_chat_opened",- triggered_from: "header",- });- H.closeMetabotViaCloseButton();- });-- it("should be controlled via keyboard shortcut", () => {+ it("should be controlled via keyboard shortcut, be able to be opened and closed, send a message to the agent, handle successful or failed responses, and start a new conversation via /metabot/new", () => {H.openMetabotViaShortcutKey();H.expectUnstructuredSnowplowEvent({event: "metabot_chat_opened",@@ -158,9 +135,22 @@ describe("Metabot UI", () => {},1,);- });- it("should allow a user to send a message to the agent and handle successful or failed responses", () => {+ H.openMetabotViaSearchButton();+ H.expectUnstructuredSnowplowEvent({+ event: "metabot_chat_opened",+ triggered_from: "header",+ });+ H.expectUnstructuredSnowplowEvent(+ {+ event: "metabot_chat_opened",+ triggered_from: "keyboard_shortcut",+ },+ 1,+ );+ H.closeMetabotViaCloseButton();++ cy.log("send a message and handle successful or failed responses");H.openMetabotViaSearchButton();H.chatMessages().should("not.exist");@@ -178,9 +168,8 @@ describe("Metabot UI", () => {H.mockMetabotResponse({ statusCode: 200, body: apiKeyInvalidResponse });H.sendMetabotMessage("Who is your favorite?");H.lastChatMessage().should("contain.text", "Something went wrong");- });- it("should allow starting a new metabot conversation via the /metabot/new", () => {+ cy.log("start a new conversation via /metabot/new");H.mockMetabotResponse({statusCode: 200,body: whoIsYourFavoriteResponse,@@ -189,14 +178,6 @@ describe("Metabot UI", () => {H.assertChatVisibility("visible");H.lastChatMessage().should("have.text", "You, but don't tell anyone.");});-- it("should not submit a prompt via /metabot/new when metabot is disabled", () => {- H.updateSetting("metabot-enabled?", false);- cy.visit("/metabot/new?q=Who%20is%20your%20favorite%3F");- cy.url().should("eq", Cypress.config().baseUrl + "/");- H.assertChatVisibility("not.visible");- cy.get("@agentReq.all").should("have.length", 0);- });});});});@@ -209,7 +190,11 @@ describe("Metabot in full-app embedding", () => {H.setupAnthropicLlmProvider();});- it("should show the metabot button when embedded-metabot-enabled? is true", () => {+ it("should show the metabot button only when embedded-metabot-enabled? is true", () => {+ cy.intercept("GET", "/api/metabot/permissions/user-permissions").as(+ "getUserPermissions",+ );+H.updateEnterpriseSettings({ "embedded-metabot-enabled?": true });H.visitFullAppEmbeddingUrl({@@ -217,11 +202,13 @@ describe("Metabot in full-app embedding", () => {qs: {},});+ cy.wait("@getUserPermissions");H.appBar().icon("metabot").should("be.visible");- });- it("should not show the metabot button when embedded-metabot-enabled? is false", () => {H.updateEnterpriseSettings({ "embedded-metabot-enabled?": false });+ cy.intercept("GET", "/api/metabot/permissions/user-permissions").as(+ "getUserPermissionsWhenDisabled",+ );H.visitFullAppEmbeddingUrl({url: `/question/${ORDERS_BY_YEAR_QUESTION_ID}`,@@ -233,6 +220,7 @@ describe("Metabot in full-app embedding", () => {cy.log("Assert metabot buttons are not rendered");H.appBar().icon("metabot").should("not.exist");+ cy.get("@getUserPermissionsWhenDisabled.all").should("have.length", 0);});});@@ -248,3 +236,38 @@ const apiKeyInvalidResponse = H.createMetabotSSEBody(H.metabotErrorPart("Anthropic API key expired or invalid"),H.metabotFinishPart("error"),);++function getContainerPadding($container: JQuery<HTMLElement>) {+ const style = getComputedStyle($container[0]);+ return {+ top: parseFloat(style.paddingTop),+ bottom: parseFloat(style.paddingBottom),+ };+}++function assertFillerReachesContainerBottom() {+ cy.findByTestId("metabot-chat-messages").should(($container) => {+ const containerRect = $container[0].getBoundingClientRect();+ const fillerRect = $container+ .find("[data-testid='metabot-message-filler']")[0]+ .getBoundingClientRect();+ expect(fillerRect.height).to.be.greaterThan(0);+ expect(fillerRect.bottom).to.be.closeTo(+ containerRect.bottom - getContainerPadding($container).bottom,+ 1,+ );+ });+}++function assertLastPromptAtContainerTop() {+ cy.findByTestId("metabot-chat-messages").should(($container) => {+ const containerRect = $container[0].getBoundingClientRect();+ const prompts = $container.find("[data-message-role='user']").toArray();+ expect(prompts).to.have.length.greaterThan(0);+ const promptRect = prompts[prompts.length - 1].getBoundingClientRect();+ expect(promptRect.top).to.be.closeTo(+ containerRect.top + getContainerPadding($container).top,+ 2,+ );+ });+}diff --git a/e2e/test/scenarios/metabot/native-sql-generation.cy.spec.ts b/e2e/test/scenarios/metabot/native-sql-generation.cy.spec.tsindex d2d722640ec..82ff7f402e4 100644--- a/e2e/test/scenarios/metabot/native-sql-generation.cy.spec.ts+++ b/e2e/test/scenarios/metabot/native-sql-generation.cy.spec.ts@@ -17,7 +17,7 @@ const rejectButton = () => cy.findByTestId("reject-proposed-changes-button");const generatingLoader = () => cy.findByLabelText("Stop generating");describe("Native SQL generation", () => {- it("should show setup guidance when metabot is not configured", () => {+ it("should show setup guidance in SQL generation and on the AI exploration page when metabot is not configured", () => {H.restore();cy.signInAsAdmin();H.activateToken("pro-self-hosted");@@ -33,6 +33,14 @@ describe("Native SQL generation", () => {cy.findByRole("button", { name: "connect to a model" }).click();cy.findByTestId("ai-provider-configuration-modal").should("be.visible");++ cy.visit("/question/ask");+ cy.url().should("include", "/question/ask");+ cy.findByRole("button", { name: "connect to a model" }).should(+ "be.visible",+ );+ cy.findByRole("button", { name: "connect to a model" }).click();+ cy.findByTestId("ai-provider-configuration-modal").should("be.visible");});describe("single db", () => {@@ -41,10 +49,9 @@ describe("Native SQL generation", () => {cy.signInAsAdmin();H.activateToken("pro-self-hosted");H.setupAnthropicLlmProvider({ apiKey: "sk-ant-api03-test-token" });- cy.intercept("POST", "/api/metabot/agent-streaming").as("agentReq");});- it("should be able to successfully generate sql", () => {+ it("should be able to successfully generate sql and correctly control the input", () => {H.startNewNativeQuestion({ query: "SELECT 1" });H.NativeEditor.get().should("be.visible");@@ -85,9 +92,9 @@ describe("Native SQL generation", () => {rejectButton().should("not.exist");H.NativeEditor.get().should("not.contain", "SELECT 1");H.NativeEditor.get().should("contain", "SELECT * FROM users");- });- it("should be able to correctly control the input", () => {+ cy.log("control the input");+ cy.visit("/");H.startNewNativeQuestion();H.NativeEditor.get().should("be.visible");@@ -127,10 +134,31 @@ describe("Native SQL generation", () => {cy.signInAsAdmin();H.activateToken("pro-self-hosted");H.setupAnthropicLlmProvider({ apiKey: "sk-ant-api03-test-token" });- cy.intercept("POST", "/api/metabot/agent-streaming").as("agentReq");});- it("should manage conversation state correctly", () => {+ it("should show error if no code_edit is received and manage conversation state correctly", () => {+ H.startNewNativeQuestion();+ H.NativeEditor.get().should("be.visible");++ toggleInlineSQLPrompt();+ inlinePromptInput().click();+ cy.realType("do something", { pressDelay: 10 });+ H.mockMetabotResponse({+ body: mockTextOnlyResponse("I can help with that!"),+ });+ generateButton().click();+ cy.wait("@metabotAgent");++ errorMessage()+ .should("be.visible")+ .and(+ "have.text",+ "Sorry, I ran into an error. Could you please try that again?",+ );+ acceptButton().should("not.exist");++ cy.log("manage conversation state");+ cy.visit("/");H.startNewNativeQuestion({ query: "SELECT 1" });H.NativeEditor.get().should("be.visible");@@ -193,8 +221,10 @@ describe("Native SQL generation", () => {// leave the page, go to new SQL page and send a new promptacceptButton().click();- cy.visit("/");- H.startNewNativeQuestion();+ cy.findByTestId("main-logo-link").click();+ H.modal().button("Discard changes").click();+ cy.location("pathname").should("eq", "/");+ H.newButton("SQL query").click();H.NativeEditor.get().should("be.visible");toggleInlineSQLPrompt();inlinePromptInput().click();@@ -215,23 +245,6 @@ describe("Native SQL generation", () => {acceptButton().should("not.exist");rejectButton().should("not.exist");});-- it("should show error if no code_edit is received", () => {- H.startNewNativeQuestion();- H.NativeEditor.get().should("be.visible");-- toggleInlineSQLPrompt();- inlinePromptInput().click();- cy.realType("do something", { pressDelay: 10 });- H.mockMetabotResponse({- body: mockTextOnlyResponse("I can help with that!"),- });- generateButton().click();- cy.wait("@metabotAgent");-- errorMessage().should("be.visible");- acceptButton().should("not.exist");- });});});