The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
feat: error code on streaming response throw/fail (#62351)
Edward Paget8207fc31815f
diff --git a/e2e/test/scenarios/data-studio/table-collection-permissions.cy.spec.ts b/e2e/test/scenarios/data-studio/table-collection-permissions.cy.spec.tsindex deb0ef5fedc..c21b93ff11c 100644--- a/e2e/test/scenarios/data-studio/table-collection-permissions.cy.spec.ts+++ b/e2e/test/scenarios/data-studio/table-collection-permissions.cy.spec.ts@@ -481,7 +481,7 @@ describe("scenarios > data studio > table collection permissions", () => {cy.signIn("nodata");H.visitQuestionAdhoc(productsQuestionDetails);- assertPermissionError();+ assertQueryPermissionError();});it("should not be able to create questions when all published tables are unpublished", () => {@@ -509,7 +509,7 @@ describe("scenarios > data studio > table collection permissions", () => {cy.signIn("nodata");H.visitQuestionAdhoc(productsQuestionDetails);- assertPermissionError();+ assertQueryPermissionError();});it("should not be able to create questions even if there are published tables", () => {@@ -544,12 +544,6 @@ function librarySidebarSection() {return H.navigationSidebar().findByRole("section", { name: "Library" });}-function assertPermissionError() {- H.main()- .findByText("Sorry, you don’t have permission to see that.")- .should("be.visible");-}-function assertQueryPermissionError() {H.main().findByText("Sorry, you don't have permission to run this query.")diff --git a/e2e/test/scenarios/permissions/sandboxing/helpers/e2e-sandboxing-helpers.ts b/e2e/test/scenarios/permissions/sandboxing/helpers/e2e-sandboxing-helpers.tsindex 030e89399a9..c23efc881ac 100644--- a/e2e/test/scenarios/permissions/sandboxing/helpers/e2e-sandboxing-helpers.ts+++ b/e2e/test/scenarios/permissions/sandboxing/helpers/e2e-sandboxing-helpers.ts@@ -551,7 +551,11 @@ export const getCardResponses = (questions: SimpleCollectionItem[]) => {expect(questions.length).to.be.greaterThan(0);return H.cypressWaitAll(questions.map((question) =>- cy.request<DatasetResponse>("POST", `/api/card/${question.id}/query`),+ cy.request<DatasetResponse>({+ method: "POST",+ url: `/api/card/${question.id}/query`,+ failOnStatusCode: false,+ }),),).then((responses) => {return { responses, questions };diff --git a/enterprise/backend/test/metabase_enterprise/advanced_permissions/query_processor/middleware/permissions_test.clj b/enterprise/backend/test/metabase_enterprise/advanced_permissions/query_processor/middleware/permissions_test.cljindex 6a6268325f9..2c4339f96ee 100644--- a/enterprise/backend/test/metabase_enterprise/advanced_permissions/query_processor/middleware/permissions_test.clj+++ b/enterprise/backend/test/metabase_enterprise/advanced_permissions/query_processor/middleware/permissions_test.clj@@ -282,15 +282,16 @@(with-download-perms-for-db! (mt/id) :none(streaming-test/do-test!"A user with no download perms for a DB receives an error response"- {:query {:database (mt/id)- :type :query- :query {:source-table (mt/id :venues)- :limit 10}}- :endpoints [:card :dataset]- :assertions {:csv (fn [results]- (is (partial=- {:error "You do not have permissions to download the results of this query."}- results)))}})+ {:query {:database (mt/id)+ :type :query+ :query {:source-table (mt/id :venues)+ :limit 10}}+ :expected-status 403+ :endpoints [:card :dataset]+ :assertions {:csv (fn [results]+ (is (partial=+ {:error "You do not have permissions to download the results of this query."}+ results)))}})(streaming-test/do-test!"An admin can always run download queries, even if the All Users group has no download permissions "@@ -305,15 +306,16 @@(with-download-perms! (mt/id) {:schemas {"PUBLIC" :none}}(streaming-test/do-test!"A user with no download perms for a schema receives an error response for download queries on that schema"- {:query {:database (mt/id)- :type :query- :query {:source-table (mt/id :venues)- :limit 10}}- :endpoints [:card :dataset]- :assertions {:csv (fn [results]- (is (partial=- {:error "You do not have permissions to download the results of this query."}- results)))}}))+ {:query {:database (mt/id)+ :type :query+ :query {:source-table (mt/id :venues)+ :limit 10}}+ :expected-status 403+ :endpoints [:card :dataset]+ :assertions {:csv (fn [results]+ (is (partial=+ {:error "You do not have permissions to download the results of this query."}+ results)))}}))(with-download-perms! (mt/id) {:schemas {"PUBLIC" {(mt/id :venues) :none(mt/id :checkins) :full@@ -321,15 +323,16 @@(mt/id :categories) :full}}}(streaming-test/do-test!"A user with no download perms for a table receives an error response for download queries on that table"- {:query {:database (mt/id)- :type :query- :query {:source-table (mt/id :venues)- :limit 10}}- :endpoints [:card :dataset]- :assertions {:csv (fn [results]- (is (partial=- {:error "You do not have permissions to download the results of this query."}- results)))}})+ {:query {:database (mt/id)+ :type :query+ :query {:source-table (mt/id :venues)+ :limit 10}}+ :expected-status 403+ :endpoints [:card :dataset]+ :assertions {:csv (fn [results]+ (is (partial=+ {:error "You do not have permissions to download the results of this query."}+ results)))}})(streaming-test/do-test!"A user with no download perms for a table still has full download perms for MBQL queries on other tables"@@ -342,12 +345,13 @@(streaming-test/do-test!"A user with no download perms for a table has no download perms for native queries on all tables"- {:query (mt/native-query {:query "SELECT * FROM checkins LIMIT 10;"})- :endpoints [:card :dataset]- :assertions {:csv (fn [results]- (is (partial=- {:error "You do not have permissions to download the results of this query."}- results)))}})))))+ {:query (mt/native-query {:query "SELECT * FROM checkins LIMIT 10;"})+ :expected-status 403+ :endpoints [:card :dataset]+ :assertions {:csv (fn [results]+ (is (partial=+ {:error "You do not have permissions to download the results of this query."}+ results)))}})))))(deftest joins-test(mt/with-full-data-perms-for-all-users!@@ -362,11 +366,12 @@{:joins [{:source-table $$categories:condition [:= $category_id 1]}]:limit 10})- :endpoints [:card :dataset]- :assertions {:csv (fn [results]- (is (partial=- {:error "You do not have permissions to download the results of this query."}- results)))}})+ :expected-status 403+ :endpoints [:card :dataset]+ :assertions {:csv (fn [results]+ (is (partial=+ {:error "You do not have permissions to download the results of this query."}+ results)))}})(streaming-test/do-test!"A user has limited downloads for a query with a join if they have limited permissions for one of the tables"@@ -425,12 +430,13 @@(fn [query](streaming-test/do-test!"A table joined to a card, with no download perms for the card, results in blocked download"- {:query query- :endpoints [:card :dataset]- :assertions {:csv (fn [results]- (is (partial=- {:error "You do not have permissions to download the results of this query."}- results)))}})))))+ {:query query+ :expected-status 403+ :endpoints [:card :dataset]+ :assertions {:csv (fn [results]+ (is (partial=+ {:error "You do not have permissions to download the results of this query."}+ results)))}})))))(deftest sandbox-card-test(testing "Do we correctly check download perms for queries that involve a sandbox? (#57861)"@@ -463,15 +469,16 @@(mt/id :checkins) :none}}}(streaming-test/do-test!"A table with sandbox and not download perms"- {:query {:database (mt/id)- :type :query- :query {:source-table (mt/id :checkins)- :limit 10}}- :endpoints [:card :dataset]- :assertions {:csv (fn [results]- (is (partial=- {:error "You do not have permissions to download the results of this query."}- results)))}})))))))+ {:query {:database (mt/id)+ :type :query+ :query {:source-table (mt/id :checkins)+ :limit 10}}+ :expected-status 403+ :endpoints [:card :dataset]+ :assertions {:csv (fn [results]+ (is (partial=+ {:error "You do not have permissions to download the results of this query."}+ results)))}})))))))(defn- do-joined-cards-with-native-query-test! [f](testing "Do we correctly apply the least permissive download perms when joining cards where one has a native query?"@@ -520,21 +527,22 @@(mt/id :users) :limited}}}(streaming-test/do-test!"Join between MBQL card (venues:full) and native card - should block due to checkins:none"- {:query {:database (mt/id)- :type :query- :query {:source-table (format "card__%d" mbql-card-id)- :joins [{:fields [[:field "ID" {:base-type :type/Integer}]]- :source-table (format "card__%d" native-card-id)- :alias "native_card"- :condition [:= [:field "ID" {:base-type :type/Integer}]- [:field "VENUE_ID" {:base-type :type/Integer, :join-alias "native_card"}]]- :strategy :left-join}]- :limit 10}}- :endpoints [:card :dataset]- :assertions {:csv (fn [results]- (is (partial=- {:error "You do not have permissions to download the results of this query."}- results)))}}))))))+ {:query {:database (mt/id)+ :type :query+ :query {:source-table (format "card__%d" mbql-card-id)+ :joins [{:fields [[:field "ID" {:base-type :type/Integer}]]+ :source-table (format "card__%d" native-card-id)+ :alias "native_card"+ :condition [:= [:field "ID" {:base-type :type/Integer}]+ [:field "VENUE_ID" {:base-type :type/Integer, :join-alias "native_card"}]]+ :strategy :left-join}]+ :limit 10}}+ :expected-status 403+ :endpoints [:card :dataset]+ :assertions {:csv (fn [results]+ (is (partial=+ {:error "You do not have permissions to download the results of this query."}+ results)))}}))))))(deftest joined-cards-with-native-query-test-3(do-joined-cards-with-native-query-test!@@ -598,15 +606,16 @@(mt/id :users) :full}}}(streaming-test/do-test!"Card with native source card - should block due to checkins:none in native source"- {:query {:database (mt/id)- :type :query- :query {:source-table (format "card__%d" mbql-card-id)- :limit 10}}- :endpoints [:card :dataset]- :assertions {:csv (fn [results]- (is (partial=- {:error "You do not have permissions to download the results of this query."}- results)))}})))))))+ {:query {:database (mt/id)+ :type :query+ :query {:source-table (format "card__%d" mbql-card-id)+ :limit 10}}+ :expected-status 403+ :endpoints [:card :dataset]+ :assertions {:csv (fn [results]+ (is (partial=+ {:error "You do not have permissions to download the results of this query."}+ results)))}})))))))(deftest card-with-native-source-card-test-3(testing "Do we correctly apply download perms when downloading from a card that uses a source card with a native query?"diff --git a/enterprise/backend/test/metabase_enterprise/data_studio/permissions/query_test.clj b/enterprise/backend/test/metabase_enterprise/data_studio/permissions/query_test.cljindex 838f3d99eb4..c747887b4bf 100644--- a/enterprise/backend/test/metabase_enterprise/data_studio/permissions/query_test.clj+++ b/enterprise/backend/test/metabase_enterprise/data_studio/permissions/query_test.clj@@ -132,7 +132,8 @@(perms/set-table-permission! all-users (mt/id :venues) :perms/create-queries :no);; Without collection permission, query should fail at database check (403)(testing "Query should fail because user has no collection permission on published table"- (is (= "You don't have permissions to do that."- (mt/with-current-user user-id- (mt/user-http-request user-id :post 403 "dataset"- (mt/mbql-query venues {:limit 1})))))))))))))+ (is (=? {:status "failed"+ :error "You do not have permissions to run this query."}+ (mt/with-current-user user-id+ (mt/user-http-request user-id :post 403 "dataset"+ (mt/mbql-query venues {:limit 1})))))))))))))diff --git a/enterprise/backend/test/metabase_enterprise/sandbox/api/card_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/api/card_test.cljindex 830da8cc596..2bf83555de8 100644--- a/enterprise/backend/test/metabase_enterprise/sandbox/api/card_test.clj+++ b/enterprise/backend/test/metabase_enterprise/sandbox/api/card_test.clj@@ -131,10 +131,11 @@(data-perms/set-table-permission! group (mt/id :venues) :perms/view-data view-perm)(data-perms/set-table-permission! group (mt/id :venues) :perms/create-queries create-perm)(testing (str "view-data: " view-perm ", create-queries: " create-perm)- (let [response (mt/user-http-request user-id :post 202 (str "card/" (u/the-id card) "/query"))]- (if should-succeed?- (is (= 1 (count (mt/rows response))))- (is (thrown? clojure.lang.ExceptionInfo (mt/rows response))))))))))))+ (if should-succeed?+ (let [response (mt/user-http-request user-id :post 202 (str "card/" (u/the-id card) "/query"))]+ (is (= 1 (count (mt/rows response)))))+ (let [response (mt/user-http-request user-id :post 403 (str "card/" (u/the-id card) "/query"))]+ (is (partial= {:error_type "missing-required-permissions"} response)))))))))))(deftest sandbox-join-permissions-test(testing "Sandboxed query can't be saved when sandboxed table is joined to a table that the current user doesn't have access to"diff --git a/enterprise/frontend/src/metabase-enterprise/tools/ErrorOverview.jsx b/enterprise/frontend/src/metabase-enterprise/tools/ErrorOverview.jsxindex 49801b4959c..5f18ade679f 100644--- a/enterprise/frontend/src/metabase-enterprise/tools/ErrorOverview.jsx+++ b/enterprise/frontend/src/metabase-enterprise/tools/ErrorOverview.jsx@@ -7,6 +7,7 @@ import {SettingsSection,} from "metabase/admin/components/SettingsSection";import CS from "metabase/css/core/index.css";+import { fetchDataOrError } from "metabase/dashboard/utils";import { CardApi } from "metabase/services";import AuditParameters from "../audit_app/components/AuditParameters";@@ -59,7 +60,8 @@ export default function ErrorOverview(props) {await Promise.all(checkedCardIds.map(- async (member) => await CardApi.query({ cardId: member }),+ async (member) =>+ await fetchDataOrError(CardApi.query({ cardId: member })),),);setRowChecked({});diff --git a/frontend/src/metabase/app-main.js b/frontend/src/metabase/app-main.jsindex 0b76ac88c88..b923e2dd8a1 100644--- a/frontend/src/metabase/app-main.js+++ b/frontend/src/metabase/app-main.js@@ -18,7 +18,6 @@ const NOT_AUTHORIZED_TRIGGERS = [/\/api\/collection\/\d+(?:\/items)?$/,/\/api\/card\/\d+$/,/\/api\/pulse\/\d+$/,- /\/api\/dataset$/,];init(mainReducers, getRoutes, (store) => {diff --git a/frontend/src/metabase/dashboard/actions/data-fetching.ts b/frontend/src/metabase/dashboard/actions/data-fetching.tsindex 90ec5b3801c..3e995d2f749 100644--- a/frontend/src/metabase/dashboard/actions/data-fetching.ts+++ b/frontend/src/metabase/dashboard/actions/data-fetching.ts@@ -330,7 +330,7 @@ export const fetchCardDataAction = createAsyncThunk<),);} else if (dashboardType === "public") {- result = await fetchDataOrError(+ result = (await fetchDataOrError(maybeUsePivotEndpoint(PublicApi.dashboardCardQuery,card,@@ -347,9 +347,9 @@ export const fetchCardDataAction = createAsyncThunk<},queryOptions,),- );+ )) as Dataset | { error: unknown };} else if (dashboardType === "embed") {- result = await fetchDataOrError(+ result = (await fetchDataOrError(maybeUsePivotEndpoint(EmbedApi.dashboardCardQuery,card,@@ -366,15 +366,15 @@ export const fetchCardDataAction = createAsyncThunk<},queryOptions,),- );+ )) as Dataset | { error: unknown };} else if (dashboardType === "transient" || dashboardType === "inline") {- result = await fetchDataOrError(+ result = (await fetchDataOrError(maybeUsePivotEndpoint(MetabaseApi.dataset,card,metadata,)({ ...datasetQuery, ignore_cache: ignoreCache }, queryOptions),- );+ )) as Dataset | { error: unknown };} else {const dashcardBeforeEditing = getDashCardBeforeEditing(getState(),@@ -407,13 +407,13 @@ export const fetchCardDataAction = createAsyncThunk<dashboard_id: dashcard.dashboard_id,dashboard_load_id: dashboardLoadId,};- result = await fetchDataOrError(+ result = (await fetchDataOrError(maybeUsePivotEndpoint(endpoint,card,metadata,)(requestBody, queryOptions),- );+ )) as Dataset | { error: unknown };}// If the request was not previously cancelled, then clear the defer for the carddiff --git a/frontend/src/metabase/dashboard/utils.ts b/frontend/src/metabase/dashboard/utils.tsindex 87b3939982d..fb0e26c6d96 100644--- a/frontend/src/metabase/dashboard/utils.ts+++ b/frontend/src/metabase/dashboard/utils.ts@@ -266,6 +266,23 @@ export async function fetchDataOrError<T>(dataPromise: Promise<T>) {try {return await dataPromise;} catch (error) {+ // For 4xx errors from streaming query endpoints, the error response body+ // contains the actual error data that should be displayed (just like the old+ // 202-with-error-in-body behavior). Treat these as successful responses.+ if (+ error &&+ typeof error === "object" &&+ "status" in error &&+ typeof error.status === "number" &&+ error.status >= 400 &&+ error.status < 500 &&+ "data" in error &&+ typeof error.data === "object"+ ) {+ // Return the error data as if it were a successful response+ return error.data;+ }+ // For 5xx errors or other errors, maintain the original behaviorreturn { error };}}diff --git a/frontend/src/metabase/dashboard/utils.unit.spec.ts b/frontend/src/metabase/dashboard/utils.unit.spec.tsindex 4752e652715..50760088d2d 100644--- a/frontend/src/metabase/dashboard/utils.unit.spec.ts+++ b/frontend/src/metabase/dashboard/utils.unit.spec.ts@@ -75,7 +75,8 @@ describe("Dashboard utils", () => {const failedFetch = Promise.reject(error);const result = await fetchDataOrError(failedFetch);- expect(result.error).toEqual(error);+ expect(result).toBeTruthy();+ expect((result as { error: unknown }).error).toEqual(error);});it("should return true if a database has model actions enabled", () => {diff --git a/frontend/src/metabase/public/containers/PublicOrEmbeddedQuestion/PublicOrEmbeddedQuestion/PublicOrEmbeddedQuestion.tsx b/frontend/src/metabase/public/containers/PublicOrEmbeddedQuestion/PublicOrEmbeddedQuestion/PublicOrEmbeddedQuestion.tsxindex d8dac69752f..e18ccf7d91b 100644--- a/frontend/src/metabase/public/containers/PublicOrEmbeddedQuestion/PublicOrEmbeddedQuestion/PublicOrEmbeddedQuestion.tsx+++ b/frontend/src/metabase/public/containers/PublicOrEmbeddedQuestion/PublicOrEmbeddedQuestion/PublicOrEmbeddedQuestion.tsx@@ -2,6 +2,7 @@ import type { Location } from "history";import { useCallback, useEffect, useState } from "react";import { useLatest, useMount } from "react-use";+import { fetchDataOrError } from "metabase/dashboard/utils";import { EmbeddingEntityContextProvider } from "metabase/embedding/context";import { useDispatch, useSelector } from "metabase/lib/redux";import { LocaleProvider } from "metabase/public/LocaleProvider";@@ -118,19 +119,21 @@ export const PublicOrEmbeddedQuestion = ({try {setResult(null);- let newResult;+ let newResult: Dataset | { error: unknown };if (token) {// embeds apply parameter values server-side- newResult = await maybeUsePivotEndpoint(- EmbedApi.cardQuery,- card,- metadataRef.current,- )({- token,- parameters: JSON.stringify(- getParameterValuesBySlug(parameters, parameterValues),- ),- });+ newResult = (await fetchDataOrError(+ maybeUsePivotEndpoint(+ EmbedApi.cardQuery,+ card,+ metadataRef.current,+ )({+ token,+ parameters: JSON.stringify(+ getParameterValuesBySlug(parameters, parameterValues),+ ),+ }),+ )) as Dataset | { error: unknown };} else if (uuid) {// public links currently apply parameters client-sideconst datasetQuery = applyParameters(@@ -140,19 +143,26 @@ export const PublicOrEmbeddedQuestion = ({[],{ sparse: true },);- newResult = await maybeUsePivotEndpoint(- PublicApi.cardQuery,- card,- metadataRef.current,- )({- uuid,- parameters: JSON.stringify(datasetQuery.parameters),- });+ newResult = (await fetchDataOrError(+ maybeUsePivotEndpoint(+ PublicApi.cardQuery,+ card,+ metadataRef.current,+ )({+ uuid,+ parameters: JSON.stringify(datasetQuery.parameters),+ }),+ )) as Dataset | { error: unknown };} else {throw { status: 404 };}- setResult(newResult);+ // If error is object it is because it was a non-query error+ if (typeof newResult.error === "object") {+ dispatch(setErrorPage(newResult.error));+ } else {+ setResult(newResult as Dataset);+ }} catch (error) {console.error("error", error);dispatch(setErrorPage(error));diff --git a/frontend/src/metabase/services.js b/frontend/src/metabase/services.jsindex 0b67bb2f7f0..6618626845d 100644--- a/frontend/src/metabase/services.js+++ b/frontend/src/metabase/services.js@@ -30,6 +30,35 @@ export const StoreApi = {tokenStatus: GET("/api/premium-features/token/status"),};+/**+ * Handles API errors for query endpoints. For 4xx errors from streaming query+ * endpoints, the error response body contains the actual error data that should+ * be displayed (just like the old 202-with-error-in-body behavior). This function+ * converts 4xx errors into successful responses with the error data.+ *+ * @param {Promise} apiPromise - The API promise to handle+ * @returns {Promise} The result or error data+ */+async function handleQueryApiError(apiPromise) {+ try {+ return await apiPromise;+ } catch (error) {+ // For 4xx errors, treat the error response body as a successful response+ // (maintaining compatibility with the old 202-with-error-in-body behavior)+ if (+ error &&+ typeof error === "object" &&+ error.status >= 400 &&+ error.status < 500 &&+ error.data+ ) {+ return error.data;+ }+ // For 5xx and other errors, re-throw+ throw error;+ }+}+// Pivot tables need extra data beyond what's described in the MBQL query itself.// To fetch that extra data we rely on specific APIs for pivot tables that mirrow the normal endpoints.// Those endpoints take the query along with `pivot_rows` and `pivot_cols` to return the subtotal data.@@ -118,29 +147,33 @@ export async function runQuestionQuery(};return [- await maybeUsePivotEndpoint(- dashboardId ? DashboardApi.cardQuery : CardApi.query,- card,- question.metadata(),- )(queryParams, {- cancelled: cancelDeferred.promise,- }),+ await handleQueryApiError(+ maybeUsePivotEndpoint(+ dashboardId ? DashboardApi.cardQuery : CardApi.query,+ card,+ question.metadata(),+ )(queryParams, {+ cancelled: cancelDeferred.promise,+ }),+ ),];}const getDatasetQueryResult = (datasetQuery) => {const datasetQueryWithParameters = { ...datasetQuery, parameters };- return maybeUsePivotEndpoint(- MetabaseApi.dataset,- card,- question.metadata(),- )(- datasetQueryWithParameters,- cancelDeferred- ? {- cancelled: cancelDeferred.promise,- }- : {},+ return handleQueryApiError(+ maybeUsePivotEndpoint(+ MetabaseApi.dataset,+ card,+ question.metadata(),+ )(+ datasetQueryWithParameters,+ cancelDeferred+ ? {+ cancelled: cancelDeferred.promise,+ }+ : {},+ ),);};diff --git a/frontend/src/metabase/services.unit.spec.ts b/frontend/src/metabase/services.unit.spec.tsindex 7b0bb589e48..daa95a7b402 100644--- a/frontend/src/metabase/services.unit.spec.ts+++ b/frontend/src/metabase/services.unit.spec.ts@@ -200,4 +200,64 @@ describe("metabase/services > runQuestionQuery", () => {expect(result).toEqual([mockResult]);});});++ describe("error handling", () => {+ it("should convert 4xx errors to successful responses with error data (saved question)", async () => {+ const question = createMockSavedQuestion();+ const errorData = {+ error: "Query failed",+ error_type: "client-error",+ status: "failed",+ };++ fetchMock.post(getQueryEndpointPath(question), {+ status: 400,+ body: errorData,+ });++ const result = await runQuestionQuery(question, {+ cancelDeferred: defer(),+ });++ // 4xx errors should be returned as successful responses with error data+ expect(result).toEqual([errorData]);+ });++ it("should convert 4xx errors to successful responses with error data (ad-hoc question)", async () => {+ const question = createMockAdHocQuestion();+ const errorData = {+ error: "Permission denied",+ error_type: "permission-error",+ status: "failed",+ };++ fetchMock.post(getQueryEndpointPath(question), {+ status: 403,+ body: errorData,+ });++ const result = await runQuestionQuery(question, {+ cancelDeferred: defer(),+ });++ // 4xx errors should be returned as successful responses with error data+ expect(result).toEqual([errorData]);+ });++ it("should throw on 5xx server errors", async () => {+ const question = createMockSavedQuestion();++ fetchMock.post(getQueryEndpointPath(question), {+ status: 500,+ body: { error: "Internal server error" },+ });++ // 5xx errors should still throw+ await expect(+ runQuestionQuery(question, { cancelDeferred: defer() }),+ ).rejects.toMatchObject({+ status: 500,+ });+ });+ });});diff --git a/src/metabase/query_processor/api.clj b/src/metabase/query_processor/api.cljindex 2712a6029a3..484efb0ee7f 100644--- a/src/metabase/query_processor/api.clj+++ b/src/metabase/query_processor/api.clj@@ -62,8 +62,7 @@(not= database lib.schema.id/saved-questions-virtual-database-id))(when-not database(throw (ex-info (tru "`database` is required for all queries whose type is not `internal`.")- {:status-code 400, :query query})))- (api/query-check :model/Database database))+ {:status-code 400, :query query}))));; store table id trivially iff we get a query with simple source-table(let [table-id (get-in query [:query :source-table])](when (int? table-id)diff --git a/src/metabase/query_processor/error_type.clj b/src/metabase/query_processor/error_type.cljindex 8c5a48ce1f9..8ee728779e6 100644--- a/src/metabase/query_processor/error_type.clj+++ b/src/metabase/query_processor/error_type.clj@@ -45,6 +45,11 @@"The current user does not have required permissions to run the current query.":parent client)+(defn permission-error?+ "Is `error-type` a permissions error"+ [error-type]+ (= missing-required-permissions error-type))+(deferror bad-configuration"Something related to configuration (e.g. of a sandbox/GTAP) is preventing us from being able to run the query.":parent client)@@ -99,6 +104,11 @@:parent invalid-query:show-in-embeds? true)+(deferror circular-reference+ "The query has circular referencing sub-queries."+ :parent invalid-query+ :show-in-embeds? true)+;;;; ### Server-Side Errors(deferror serverdiff --git a/src/metabase/query_processor/middleware/resolve_referenced.clj b/src/metabase/query_processor/middleware/resolve_referenced.cljindex 2a7b7e8ab69..bc28273fc93 100644--- a/src/metabase/query_processor/middleware/resolve_referenced.clj+++ b/src/metabase/query_processor/middleware/resolve_referenced.clj@@ -118,7 +118,8 @@[node-type node-id] node[dependency-type dependency-id] dependency](if (= reason :weavejester.dependency/circular-dependency)- (throw (ex-info (circular-ref-error query node-type node-id dependency-type dependency-id) {:original-exception e}))+ (throw (ex-info (circular-ref-error query node-type node-id dependency-type dependency-id) {:original-exception e+ :type qp.error-type/circular-reference}))(throw e))))))(mu/defn resolve-referenced-card-resources :- ::lib.schema/querydiff --git a/src/metabase/query_processor/streaming.clj b/src/metabase/query_processor/streaming.cljindex 9ea8a2a450a..217965b98ab 100644--- a/src/metabase/query_processor/streaming.clj+++ b/src/metabase/query_processor/streaming.clj@@ -5,6 +5,7 @@[metabase.lib.core :as lib][metabase.lib.schema.common :as lib.schema.common][metabase.models.visualization-settings :as mb.viz]+ [metabase.query-processor.error-type :as qp.error-type][metabase.query-processor.pipeline :as qp.pipeline][metabase.query-processor.schema :as qp.schema][metabase.query-processor.streaming.csv :as qp.csv]@@ -213,6 +214,24 @@(binding [qp.pipeline/*result* (streaming-result-fn results-writer os)](f rff))))+(defn- status-code+ "Get a status code for the supplied error object."+ [err]+ (cond+ ;; If the error is setting its own status code use that+ (:status-code err) (:status-code err)+ ;; If the error is setting its own status code use that+ (-> err :ex-data :status-code) (-> err :ex-data :status-code)+ ;; If this is a permission error return 403+ (-> err :error_type qp.error-type/permission-error?)+ 403+ ;; If this is a client error return 400+ (-> err :error_type qp.error-type/client-error?)+ 400+ ;; Use 500 for all other error statuses+ :else+ 500))+(defn -streaming-response"Impl for [[streaming-response]]."^StreamingResponse [export-format filename-prefix f]@@ -238,7 +257,7 @@(assert (not (instance? ManyToManyChannel result)) "QP should not return a core.async channel.")(when (or (instance? Throwable result)(= (:status result) :failed))- (streaming-response/write-error! os result export-format)))))))))+ (streaming-response/write-error! os result export-format (status-code result))))))))))(defn transforming-query-response"Decorate the streaming rff to transform the top-level payload."diff --git a/src/metabase/server/streaming_response.clj b/src/metabase/server/streaming_response.cljindex f503780643d..064a166318e 100644--- a/src/metabase/server/streaming_response.clj+++ b/src/metabase/server/streaming_response.clj@@ -48,36 +48,84 @@(cond-> (assoc (Throwable->map e) :_status (ex-status-code e))(server.settings/hide-stacktraces) (dissoc :via :trace)))+(def ^:dynamic *response*+ "The `HttpServletResponse` for the current streaming response.+ Bound automatically inside `streaming-response` bodies in the Jetty async path.+ Use the helper functions [[committed?]], [[set-status!]], [[set-header!]], and+ [[set-content-type!]] to interact with it."+ nil)++(defn- assert-response-bound! []+ (when-not *response*+ (throw (ex-info "Cannot call response control functions outside of a streaming-response context"+ {}))))++(defn committed?+ "Returns true if the HTTP response has already been committed (headers sent to client).+ Raises if called outside a `streaming-response` context."+ []+ (assert-response-bound!)+ (.isCommitted ^HttpServletResponse *response*))++(defn set-status!+ "Set the HTTP status code on the response. No-op if the response is already committed.+ Raises if called outside a `streaming-response` context."+ [code]+ (assert-response-bound!)+ (when-not (committed?)+ (.setStatus ^HttpServletResponse *response* (int code))))++(defn set-header!+ "Set a header on the HTTP response. No-op if the response is already committed.+ Raises if called outside a `streaming-response` context."+ [name value]+ (assert-response-bound!)+ (when-not (committed?)+ (.setHeader ^HttpServletResponse *response* (str name) (str value))))++(defn set-content-type!+ "Set the Content-Type on the HTTP response. No-op if the response is already committed.+ Raises if called outside a `streaming-response` context."+ [ct]+ (assert-response-bound!)+ (when-not (committed?)+ (.setContentType ^HttpServletResponse *response* (str ct))))+(defn write-error!"Write an error to the output stream, formatting it nicely. Closes output stream afterwards."- [^OutputStream os obj export-format]- (cond- (some #(instance? % obj)- [InterruptedException EofException])- (log/trace "Error is an InterruptedException or EofException, not writing to output stream")-- (instance? Throwable obj)- (recur os (format-exception obj) export-format)-- :else- (with-open [os os]- (log/trace (u/pprint-to-str (list 'write-error! obj)))- (try- (let [obj (-> (if (not= :api export-format)- (walk/prewalk- (fn [x]- (if (map? x)- (apply dissoc x [:json_query :preprocessed])- x))+ ([os obj export-format]+ (write-error! os obj export-format nil))+ ([^OutputStream os obj export-format status-code]+ (when (and *response* (not (committed?)))+ (set-status! (or status-code 500))+ (set-content-type! "application/json"))+ (cond+ (some #(instance? % obj)+ [InterruptedException EofException])+ (log/trace "Error is an InterruptedException or EofException, not writing to output stream")++ (instance? Throwable obj)+ (recur os (format-exception obj) export-format status-code)++ :else+ (with-open [os os]+ (log/trace (u/pprint-to-str (list 'write-error! obj)))+ (try+ (let [obj (-> (if (not= :api export-format)+ (walk/prewalk+ (fn [x]+ (if (map? x)+ (apply dissoc x [:json_query :preprocessed])+ x))+ obj)obj)- obj)- (dissoc :export-format)- (cond-> (server.settings/hide-stacktraces) (dissoc :stacktrace :trace :via)))]- (with-open [writer (BufferedWriter. (OutputStreamWriter. os StandardCharsets/UTF_8))]- (json/encode-to obj writer {})))- (catch EofException _)- (catch Throwable e- (log/error e "Error writing error to output stream" obj))))))+ (dissoc :export-format)+ (cond-> (server.settings/hide-stacktraces) (dissoc :stacktrace :trace :via)))]+ (with-open [writer (BufferedWriter. (OutputStreamWriter. os StandardCharsets/UTF_8))]+ (json/encode-to obj writer {})))+ (catch EofException _)+ (catch Throwable e+ (log/error e "Error writing error to output stream" obj)))))))(defn- do-f* [f ^OutputStream os _finished-chan canceled-chan](try@@ -107,25 +155,26 @@(defn- do-f-async"Runs `f` asynchronously on the streaming response `thread-pool`, returning immediately. When `f` finishes,completes (i.e., closes) Jetty `async-context`."- [^AsyncContext async-context f ^OutputStream os finished-chan canceled-chan]+ [^AsyncContext async-context response f ^OutputStream os finished-chan canceled-chan]{:pre [(some? os)]}(let [task (^:once fn* []- (try- (do-f* f os finished-chan canceled-chan)- (catch Throwable e- (log/error e "Caught unexpected Exception in streaming response body")- (a/>!! finished-chan :unexpected-error)- (write-error! os e nil))- (finally- ;; Clear the interrupted flag to prevent the thread from- ;; carrying stale interrupted state to the next task.- (Thread/interrupted)- (a/>!! finished-chan (if (a/poll! canceled-chan)- :canceled- :completed))- (a/close! finished-chan)- (a/close! canceled-chan)- (.complete async-context))))+ (binding [*response* response]+ (try+ (do-f* f os finished-chan canceled-chan)+ (catch Throwable e+ (log/error e "Caught unexpected Exception in streaming response body")+ (a/>!! finished-chan :unexpected-error)+ (write-error! os e nil))+ (finally+ ;; Clear the interrupted flag to prevent the thread from+ ;; carrying stale interrupted state to the next task.+ (Thread/interrupted)+ (a/>!! finished-chan (if (a/poll! canceled-chan)+ :canceled+ :completed))+ (a/close! finished-chan)+ (a/close! canceled-chan)+ (.complete async-context)))))fut (.submit (thread-pool/thread-pool) ^Runnable task)](start-interrupt-escalation! fut finished-chan canceled-chan)nil))@@ -261,7 +310,7 @@(let [output-stream-delay (output-stream-delay gzip? response)delay-os (delay-output-stream output-stream-delay)](start-async-cancel-loop! request finished-chan canceled-chan)- (do-f-async async-context f delay-os finished-chan canceled-chan)))+ (do-f-async async-context response f delay-os finished-chan canceled-chan)))(catch Throwable e(log/error e "Unexpected exception in do-f-async")(try@@ -332,6 +381,9 @@NOTE: `canceled-chan` **IS NOT WORKING**; see `metabase.server.streaming-response-test/canceling-response-2``canceled-chan` can be monitored to see if the request is canceled before results are fully written to the stream.+ Inside the body, [[*response*]] is bound to the `HttpServletResponse`. You can use the helper functions+ [[set-status!]], [[set-header!]], [[set-content-type!]], and [[committed?]] to interact with the response.+Current options:* `:content-type` -- string content type to return in the results. This is required!diff --git a/test/metabase/api/downloads_exports_test.clj b/test/metabase/api/downloads_exports_test.cljindex 3369d82cc5f..4b886c5f0e9 100644--- a/test/metabase/api/downloads_exports_test.clj+++ b/test/metabase/api/downloads_exports_test.clj@@ -1119,7 +1119,7 @@;; for now, don't try to read xlsx back in, it will not be correct since we end up writing;; a json blob to the output stream, it creates an invalid xlsx anyway.;; This is not new behaviour, we'll just fix it when a better solution to 'errors in downloaded files' comes along- (let [results (mt/user-http-request :rasta :post 200 (format "card/%d/query/%s" card-id export-format)+ (let [results (mt/user-http-request :rasta :post 500 (format "card/%d/query/%s" card-id export-format){:format_rows true})results-string (if (= "xlsx" export-format)(read-xlsx results)diff --git a/test/metabase/documents/api/document_test.clj b/test/metabase/documents/api/document_test.cljindex 5183da1d354..34ec3798c1b 100644--- a/test/metabase/documents/api/document_test.clj+++ b/test/metabase/documents/api/document_test.clj@@ -2347,7 +2347,7 @@;; Grant collection read permissions so user can access the document(perms/grant-collection-read-permissions! all-users-group coll-id)- (testing "User without download permissions yields permissions error in body (streaming uses 200)"+ (testing "User without download permissions yields permissions error";; Set download permissions to :no (no downloads allowed) for All Users group(data-perms/set-database-permission! all-users-group (mt/id) :perms/download-results :no)@@ -2357,7 +2357,7 @@[:ex-data [:map[:permissions-error? [:= true]]]]](mt/user-http-request :rasta- :post 200+ :post 403(format "document/%s/card/%s/query/csv" doc-id card-id){:parameters []:format_rows falsediff --git a/test/metabase/embedding_rest/api/embed_test.clj b/test/metabase/embedding_rest/api/embed_test.cljindex 1eb820468d1..41eb44b42df 100644--- a/test/metabase/embedding_rest/api/embed_test.clj+++ b/test/metabase/embedding_rest/api/embed_test.clj@@ -343,7 +343,7 @@(mt/test-helpers-set-global-values!(do-response-formats [response-format _request-options](with-embedding-enabled-and-new-secret-key!- (let [expected-status (response-format->status-code response-format)]+ (let [expected-status 400](testing (str "If the card has an invalid query we should just get a generic \"query failed\" ""exception (rather than leaking query info)")(with-temp-card [card {:enable_embedding true, :dataset_query {:database (mt/id)@@ -862,7 +862,7 @@(is (= {:status "failed":error "An error occurred while running the query.":error_type "invalid-query"}- (client/client :get 202 (dashcard-url dashcard))))))))+ (client/client :get 400 (dashcard-url dashcard))))))))(deftest check-that-the-dashcard-endpoint-doesn-t-work-if-embedding-isn-t-enabled(mt/with-temporary-setting-values [enable-embedding false]diff --git a/test/metabase/public_sharing_rest/api_test.clj b/test/metabase/public_sharing_rest/api_test.cljindex 9ad5c317216..8141534b7f8 100644--- a/test/metabase/public_sharing_rest/api_test.clj+++ b/test/metabase/public_sharing_rest/api_test.clj@@ -507,7 +507,7 @@(is (= {:status "failed":error "You'll need to pick a value for 'Price' before this query can run.":error_type "missing-required-parameter"}- (client/client :get 202 (str "public/card/" uuid "/query")))))))+ (client/client :get 400 (str "public/card/" uuid "/query")))))))(defn- card-with-date-field-filter [](assoc (shared-obj)diff --git a/test/metabase/queries_rest/api/card_test.clj b/test/metabase/queries_rest/api/card_test.cljindex 21b3e2701a3..95806fd82a8 100644--- a/test/metabase/queries_rest/api/card_test.clj+++ b/test/metabase/queries_rest/api/card_test.clj@@ -304,8 +304,8 @@(let [;; any strings will work here (must be shorter than 254 chars), but these are semi-relaistic:client-string (mt/random-name)version-string (str "1." (rand-int 1000) "." (rand-int 1000))]- (mt/with-temp [:model/Database {database-id :id} {}- :model/Card card-1 {:name "Card 1" :database_id database-id}]+ (mt/with-temp [:model/Card card-1 {:name "Card 1"+ :dataset_query (mt/mbql-query venues {:limit 1})}](mt/with-premium-features #{:audit-app}(mt/user-http-request :crowberto :post 202 (str "card/" (u/the-id card-1) "/query"){:request-options {:headers {"x-metabase-client" client-string@@ -2620,7 +2620,8 @@(deftest ^:parallel download-response-headers-test(testing "Make sure CSV/etc. download requests come back with the correct headers"- (mt/with-temp [:model/Card card {:name "My Awesome Card"}]+ (mt/with-temp [:model/Card card {:name "My Awesome Card"+ :dataset_query (mt/mbql-query venues {:limit 1})}](is (= {"Cache-Control" "max-age=0, no-cache, must-revalidate, proxy-revalidate""Content-Disposition" "attachment; filename=\"my_awesome_card_<timestamp>.csv\"""Content-Type" "text/csv"@@ -3824,6 +3825,19 @@(is (mi/can-read? card)))(is (= [[1] [2]] (mt/rows (process-query))))))))))+(deftest blocked-database-permissions-card-query-test+ (testing "POST /api/card/:id/query should return an error when the user has blocked view-data permissions on the database (OSS)"+ (mt/with-premium-features #{}+ (mt/with-temp-copy-of-db+ (mt/with-no-data-perms-for-all-users!+ (data-perms/set-database-permission! (perms-group/all-users) (mt/id) :perms/view-data :blocked)+ (data-perms/set-database-permission! (perms-group/all-users) (mt/id) :perms/create-queries :no)+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query venues {:limit 1})}]+ (is (malli= [:map+ [:status [:= "failed"]]+ [:error_type [:= "missing-required-permissions"]]]+ (mt/user-http-request :rasta :post 403 (format "card/%d/query" card-id))))))))))+(defn- native-card-with-template-tags []{:dataset_query{:type :nativediff --git a/test/metabase/query_processor/api_test.clj b/test/metabase/query_processor/api_test.cljindex bf2b27b8b0c..fc4a0446713 100644--- a/test/metabase/query_processor/api_test.clj+++ b/test/metabase/query_processor/api_test.clj@@ -135,13 +135,13 @@;; clear out recent query executions!(t2/delete! :model/QueryExecution)(testing "POST /api/dataset"- (testing "\nEven if a query fails we still expect a 202 response from the API"+ (testing "\nA failed query should return a 400 response from the API";; Error message's format can differ a bit depending on DB version and the comment we prepend to it, so check;; that it exists and contains the substring "Syntax error in SQL statement"(let [query {:database (mt/id):type "native":native {:query "foobar"}}- result (mt/user-http-request :crowberto :post 202 "dataset" query)]+ result (mt/user-http-request :crowberto :post 400 "dataset" query)](testing "\nAPI Response"(is (malli= [:map[:data [:map@@ -379,6 +379,19 @@(mt/user-http-request :rasta :post "dataset"(mt/mbql-query venues {:limit 1}))))))))+(deftest blocked-database-permissions-test+ (testing "POST /api/dataset should return an error when the user has blocked view-data permissions on the database (OSS)"+ (mt/with-premium-features #{}+ (mt/with-temp-copy-of-db+ (mt/with-no-data-perms-for-all-users!+ (perms/set-database-permission! (perms/all-users-group) (mt/id) :perms/view-data :blocked)+ (perms/set-database-permission! (perms/all-users-group) (mt/id) :perms/create-queries :no)+ (is (malli= [:map+ [:status [:= "failed"]]+ [:error [:= "You do not have permissions to run this query."]]]+ (mt/user-http-request :rasta :post "dataset"+ (mt/mbql-query venues {:limit 1})))))))))+(deftest api-card-join-permissions-test(testing "POST /api/dataset should error for card join permission violations"(mt/with-non-admin-groups-no-root-collection-perms@@ -1096,12 +1109,14 @@;; Set table-level permissions(perms/set-table-permission! all-users (mt/id :venues) :perms/view-data :unrestricted)(perms/set-table-permission! all-users (mt/id :venues) :perms/create-queries :no)- ;; In OSS: published tables don't grant database access, so user gets 403 at database check+ ;; In OSS: published tables don't grant database access, so user gets a permission error(testing "Query should be blocked because OSS doesn't grant database access via published tables"- (is (= "You don't have permissions to do that."- (mt/with-current-user user-id- (mt/user-http-request user-id :post 403 "dataset"- (mt/mbql-query venues {:limit 1})))))))))))))+ (is (malli= [:map+ [:status [:= "failed"]]+ [:error [:= "You do not have permissions to run this query."]]]+ (mt/with-current-user user-id+ (mt/user-http-request user-id :post "dataset"+ (mt/mbql-query venues {:limit 1})))))))))))))(deftest query-metadata-sensitive-fields-test(testing "POST /api/dataset/query_metadata"diff --git a/test/metabase/query_processor/middleware/permissions_test.clj b/test/metabase/query_processor/middleware/permissions_test.cljindex fa941ca84c0..7a251f00212 100644--- a/test/metabase/query_processor/middleware/permissions_test.clj+++ b/test/metabase/query_processor/middleware/permissions_test.clj@@ -729,7 +729,6 @@(deftest e2e-ignore-user-supplied-card-ids-test(testing "You shouldn't be able to bypass security restrictions by passing `[:info :card-id]` in the query."(mt/with-temp-copy-of-db- ;; TODO: re-evaluate this test; the error is being thrown at the API-layer and not in the QP(mt/with-no-data-perms-for-all-users!(mt/with-restored-data-perms-for-group! (u/the-id (perms/all-users-group))(mt/with-temp [:model/Collection collection {}@@ -738,9 +737,10 @@;; Since the collection derives from the root collection this grant shouldn't really be needed, but better to;; be extra-sure in this case that the user is getting rejected for data perms and not card/collection perms(perms/grant-collection-read-permissions! (perms/all-users-group) collection)- (is (= "You don't have permissions to do that."- (mt/user-http-request :rasta :post 403 "dataset" (assoc (mt/mbql-query venues {:limit 1})- :info {:card-id (u/the-id card)}))))))))))+ (is (=? {:status "failed"+ :error "You do not have permissions to run this query."}+ (mt/user-http-request :rasta :post 403 "dataset" (assoc (mt/mbql-query venues {:limit 1})+ :info {:card-id (u/the-id card)}))))))))))(deftest e2e-ignore-user-supplied-perms-test(testing "You shouldn't be able to bypass security restrictions by passing in `:query-permissions/perms` in the query"diff --git a/test/metabase/query_processor/streaming_test.clj b/test/metabase/query_processor/streaming_test.cljindex 700c0ff9656..71096431122 100644--- a/test/metabase/query_processor/streaming_test.clj+++ b/test/metabase/query_processor/streaming_test.clj@@ -317,9 +317,10 @@(defn do-test!"Test helper to enable writing API-level export tests across multiple export endpoints and formats."- [message {:keys [query viz-settings assertions endpoints user]}]+ [message {:keys [query viz-settings assertions endpoints user expected-status]}](testing message- (let [query-json (json/encode query)+ (let [expected-status (or expected-status 200)+ query-json (json/encode query)viz-settings-json (some-> viz-settings json/encode)public-uuid (str (random-uuid))card-defaults {:dataset_query query, :public_uuid public-uuid, :enable_embedding true}@@ -339,7 +340,7 @@(testing endpoint(case endpoint:dataset- (let [results (mt/user-http-request user :post 200+ (let [results (mt/user-http-request user :post expected-status(format "dataset/%s" (name export-format)){:request-options {:as (if (= export-format :xlsx) :byte-array :string)}}{:format_rows true@@ -348,14 +349,14 @@((-> assertions export-format) results)):card- (let [results (mt/user-http-request user :post 200+ (let [results (mt/user-http-request user :post expected-status(format "card/%d/query/%s" (u/the-id card) (name export-format)){:request-options {:as (if (= export-format :xlsx) :byte-array :string)}}{:format_rows true})]((-> assertions export-format) results)):dashboard- (let [results (mt/user-http-request user :post 200+ (let [results (mt/user-http-request user :post expected-status(format "dashboard/%d/dashcard/%d/card/%d/query/%s"(u/the-id dashboard)(u/the-id dashcard)@@ -367,13 +368,13 @@;; TODO -- what about the public dashcard endpoint???:public- (let [results (mt/user-http-request user :get 200+ (let [results (mt/user-http-request user :get expected-status(format "public/card/%s/query/%s?format_rows=true" public-uuid (name export-format)){:request-options {:as (if (= export-format :xlsx) :byte-array :string)}})]((-> assertions export-format) results)):embed- (let [results (mt/user-http-request user :get 200+ (let [results (mt/user-http-request user :get expected-status(embed-test/card-query-url card (str "/" (name export-format))){:request-options {:as (if (= export-format :xlsx) :byte-array :string)}})]((-> assertions export-format) results)))))))))))diff --git a/test/metabase/server/streaming_response_test.clj b/test/metabase/server/streaming_response_test.cljindex e38c399ec5b..2e6ff5f63cc 100644--- a/test/metabase/server/streaming_response_test.clj+++ b/test/metabase/server/streaming_response_test.clj@@ -20,7 +20,7 @@(:import(jakarta.servlet AsyncContext ServletOutputStream)(jakarta.servlet.http HttpServletResponse)- (java.io Closeable InputStream)+ (java.io ByteArrayOutputStream Closeable InputStream)(java.util.concurrent Executors Future)(org.apache.commons.lang3.concurrent BasicThreadFactory$Builder)))@@ -101,21 +101,21 @@(mt/user-http-request :lucky:post 202 "dataset"{:database (mt/id)- :type "native"- :native {:query {:sleep 10}}})))))))+ :type "native"+ :native {:query {:sleep 10}}})))))))(deftest truly-async-test(testing "StreamingResponses should truly be asynchronous, and not block Jetty threads while waiting for results"(with-test-driver-db!- (let [num-requests (+ thread-pool-size 20)- remaining (atom num-requests)- session-token (client/authenticate (mt/user->credentials :lucky))- url (client/build-url "dataset" nil)- request (client/build-request-map session-token- {:database (mt/id)- :type "native"- :native {:query {:sleep 2000}}}- nil)]+ (let [num-requests (+ thread-pool-size 20)+ remaining (atom num-requests)+ session-token (client/authenticate (mt/user->credentials :lucky))+ url (client/build-url "dataset" nil)+ request (client/build-request-map session-token+ {:database (mt/id)+ :type "native"+ :native {:query {:sleep 2000}}}+ nil)](testing (format "%d simultaneous queries" num-requests)(dotimes [_ num-requests](future (http/post url request)))@@ -136,14 +136,14 @@(with-test-driver-db!(reset! canceled? false)(with-start-execution-chan [start-chan]- (let [url (client/build-url "dataset" nil)+ (let [url (client/build-url "dataset" nil)session-token (client/authenticate (mt/user->credentials :lucky))- request (client/build-request-map session-token- {:database (mt/id)- :type "native"- :native {:query {:sleep 5000}}}- nil)- futur (http/post url (assoc request :async? true) identity (fn [e] (throw e)))]+ request (client/build-request-map session-token+ {:database (mt/id)+ :type "native"+ :native {:query {:sleep 5000}}}+ nil)+ futur (http/post url (assoc request :async? true) identity (fn [e] (throw e)))](is (future? futur));; wait a little while for the query to start running -- this should usually happen fairly quickly(mt/wait-for-result start-chan (u/seconds->ms 15))@@ -210,18 +210,18 @@(let [streaming-response (binding [*number-of-cans* 2](streaming-response/streaming-response nil [os _](.write os (.getBytes (format "%s cans" *number-of-cans*) "UTF-8"))))- complete-promise (promise)]+ complete-promise (promise)](server.protocols/respond streaming-response- {:response (reify HttpServletResponse- (setStatus [_ _])- (setHeader [_ _ _])- (getOutputStream [_]- (proxy [ServletOutputStream] []- (write- ([byytes]- (.write os ^bytes byytes))- ([byytes offset length]- (.write os ^bytes byytes offset length))))))+ {:response (reify HttpServletResponse+ (setStatus [_ _])+ (setHeader [_ _ _])+ (getOutputStream [_]+ (proxy [ServletOutputStream] []+ (write+ ([byytes]+ (.write os ^bytes byytes))+ ([byytes offset length]+ (.write os ^bytes byytes offset length)))))):async-context (reify AsyncContext(complete [_](deliver complete-promise true)))})@@ -230,6 +230,124 @@(is (= "2 cans"(String. (.toByteArray os) "UTF-8")))))))+(deftest write-error-uncommitted-response-test+ (testing "write-error! should set status and content type when response is not committed"+ (let [os (ByteArrayOutputStream.)+ status-called (atom nil)+ content-type-called (atom nil)]+ (binding [streaming-response/*response*+ (reify HttpServletResponse+ (isCommitted [_] false)+ (setStatus [_ status] (reset! status-called status))+ (setContentType [_ ct] (reset! content-type-called ct)))]+ (streaming-response/write-error! os {:error "test error"} :api 400))+ (testing "Status should be set to provided status code"+ (is (= 400 @status-called)))+ (testing "Content type should be set to application/json"+ (is (= "application/json" @content-type-called))))))++(deftest write-error-committed-response-test+ (testing "write-error! should not set status or content type when response is committed"+ (let [os (ByteArrayOutputStream.)+ status-called (atom nil)+ content-type-called (atom nil)]+ (binding [streaming-response/*response*+ (reify HttpServletResponse+ (isCommitted [_] true)+ (setStatus [_ status] (reset! status-called status))+ (setContentType [_ ct] (reset! content-type-called ct)))]+ (streaming-response/write-error! os {:error "test error"} :api 400))+ (testing "Status should not be set when response is committed"+ (is (nil? @status-called)))+ (testing "Content type should not be set when response is committed"+ (is (nil? @content-type-called))))))++(deftest write-error-no-response-test+ (testing "write-error! should not attempt to set status when no *response* is bound"+ (let [os (ByteArrayOutputStream.)]+ (binding [streaming-response/*response* nil]+ ;; Should not throw exception when no response is bound+ (is (some? (streaming-response/write-error! os {:error "test error"} :api 500)))))))++(deftest set-status-outside-streaming-context-test+ (testing "Calling set-status! outside a streaming-response context should raise"+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"Cannot call response control functions outside of a streaming-response context"+ (streaming-response/set-status! 500)))))++(deftest committed?-outside-streaming-context-test+ (testing "Calling committed? outside a streaming-response context should raise"+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"Cannot call response control functions outside of a streaming-response context"+ (streaming-response/committed?)))))++(deftest write-error-default-status-test+ (testing "write-error! should use default status 500 when no status provided"+ (let [os (ByteArrayOutputStream.)+ status-called (atom nil)]+ (binding [streaming-response/*response*+ (reify HttpServletResponse+ (isCommitted [_] false)+ (setStatus [_ status] (reset! status-called status))+ (setContentType [_ _]))]+ (streaming-response/write-error! os {:error "test error"} :api))+ (testing "Status should default to 500 when not provided"+ (is (= 500 @status-called))))))++(deftest write-error-exception-handling-test+ (testing "write-error! should handle different exception types appropriately"+ (let [os (ByteArrayOutputStream.)]+ (binding [streaming-response/*response*+ (reify HttpServletResponse+ (isCommitted [_] false)+ (setStatus [_ _])+ (setContentType [_ _]))]+ (testing "InterruptedException should not write to output stream"+ (streaming-response/write-error! os (InterruptedException. "interrupted") :api)+ (is (zero? (.size os))))++ (testing "EofException should not write to output stream"+ (.reset os)+ (streaming-response/write-error! os (org.eclipse.jetty.io.EofException. "eof") :api)+ (is (zero? (.size os))))++ (testing "Other exceptions should be formatted and written"+ (.reset os)+ (streaming-response/write-error! os (RuntimeException. "runtime error") :api)+ (is (pos? (.size os)))+ (let [output (String. (.toByteArray os) "UTF-8")]+ (is (re-find #"runtime error" output))))))))++(deftest write-error-json-output-test+ (testing "write-error! should write valid JSON to output stream"+ (let [os (ByteArrayOutputStream.)]+ (binding [streaming-response/*response*+ (reify HttpServletResponse+ (isCommitted [_] false)+ (setStatus [_ _])+ (setContentType [_ _]))]+ (streaming-response/write-error! os {:error "test error" :code 123} :api))+ (let [output (String. (.toByteArray os) "UTF-8")]+ (is (re-find #"\"error\":\s*\"test error\"" output))+ (is (re-find #"\"code\":\s*123" output))))))++(deftest write-error-non-api-format-test+ (testing "write-error! should strip sensitive fields for non-API export formats"+ (let [os (ByteArrayOutputStream.)]+ (binding [streaming-response/*response*+ (reify HttpServletResponse+ (isCommitted [_] false)+ (setStatus [_ _])+ (setContentType [_ _]))]+ (streaming-response/write-error! os {:error "test error"+ :json_query "SELECT * FROM table"+ :preprocessed "some data"} :csv))+ (let [output (String. (.toByteArray os) "UTF-8")]+ (is (re-find #"\"error\":\s*\"test error\"" output))+ (is (not (re-find #"json_query" output)))+ (is (not (re-find #"preprocessed" output)))))))(deftest write-error-includes-stacktrace-when-hide-stacktraces-disabled-test(testing "write-error! includes stacktrace and exception chain when hide-stacktraces is false"(mt/with-temporary-setting-values [hide-stacktraces false]@@ -413,6 +531,7 @@(reify AsyncContext(complete [_](deliver complete-promise true)))+ nil(fn [_os _canceled-chan](deliver task-started? true)(trydiff --git a/test/metabase/test/http_client.clj b/test/metabase/test/http_client.cljindex 9a6f7aa2677..7999b4a2c31 100644--- a/test/metabase/test/http_client.clj+++ b/test/metabase/test/http_client.clj@@ -12,6 +12,7 @@[medley.core :as m][metabase.config.core :as config][metabase.server.middleware.session :as mw.session]+ [metabase.server.streaming-response :as streaming-response][metabase.server.test-handler :as server.test-handler][metabase.test-runner.assert-exprs :as test-runner.assert-exprs][metabase.test.initialize :as initialize]@@ -26,6 +27,7 @@[ring.util.codec :as codec][ring.util.mime-type :as mime-type])(:import+ (jakarta.servlet.http HttpServletResponse)(java.io ByteArrayInputStream ByteArrayOutputStream File InputStream)(java.nio.charset Charset)(metabase.server.streaming_response StreamingResponse)@@ -317,36 +319,45 @@(update response :body parse-response)))(defn- read-streaming-response+ "Read a streaming response, returning a map with `:body` and optionally `:status` if the streaming body+ changed it (e.g. via `write-error!` setting a 4xx/5xx status)."[streaming-response content-type](with-open [os (java.io.ByteArrayOutputStream.)]- (let [f (.f ^StreamingResponse streaming-response)- canceled-chan (a/promise-chan)]- (f os canceled-chan)- (cond-> (.toByteArray os)- (some #(re-find % content-type) [#"json" #"text"])- (String. "UTF-8")))))+ (let [f (.f ^StreamingResponse streaming-response)+ canceled-chan (a/promise-chan)+ status-atom (atom nil)+ mock-response (reify HttpServletResponse+ (isCommitted [_] false)+ (setStatus [_ status] (reset! status-atom status))+ (setContentType [_ _])+ (setHeader [_ _ _]))]+ (binding [streaming-response/*response* mock-response]+ (f os canceled-chan))+ {:body (cond-> (.toByteArray os)+ (some #(re-find % content-type) [#"json" #"text"])+ (String. "UTF-8"))+ :status @status-atom})))(defn- coerce-mock-response-body[response]- (update response- :body- (fn [body]- (cond- ;; read the text response- (instance? InputStream body)- (with-open [r (io/reader body)]- (slurp r))-- ;; read byte array stuffs like image- (instance? (Class/forName "[B") body)- (String. ^bytes body "UTF-8")-- ;; Most APIs that execute a request returns a streaming response- (instance? StreamingResponse body)- (read-streaming-response body (get-in response [:headers "Content-Type"]))-- :else- body))))+ (let [body (:body response)]+ (cond+ (instance? InputStream body)+ (update response :body (fn [b] (with-open [r (io/reader b)] (slurp r))))++ (instance? (Class/forName "[B") body)+ (update response :body (fn [b] (String. ^bytes b "UTF-8")))++ ;; Most APIs that execute a request returns a streaming response.+ ;; read-streaming-response returns {:body ..., :status ...} where :status may be+ ;; non-nil if the streaming body set it (e.g. via write-error! on permission failure).+ (instance? StreamingResponse body)+ (let [{:keys [body status]} (read-streaming-response body (get-in response [:headers "Content-Type"]))]+ (cond-> (assoc response :body body)+ status (assoc :status status)))++ :else+ response)))(defn- build-mock-request[{:keys [query-parameters url credentials http-body method request-options]}]diff --git a/test/metabase/view_log/events/view_log_test.clj b/test/metabase/view_log/events/view_log_test.cljindex 1147dfbb26d..e89dda10b85 100644--- a/test/metabase/view_log/events/view_log_test.clj+++ b/test/metabase/view_log/events/view_log_test.clj@@ -218,7 +218,8 @@(deftest get-card-view-log-test(mt/with-premium-features #{:audit-app}(testing "Card reads (views) via the API are recorded in the view_log"- (mt/with-temp [:model/Card card {:name "My Cool Card" :type :question}]+ (mt/with-temp [:model/Card card {:name "My Cool Card" :type :question+ :dataset_query (mt/mbql-query venues {:limit 1})}](testing "POST /api/card/:id/query"(mt/user-http-request :crowberto :post 202 (format "card/%s/query" (u/id card)))(is (partial= {:user_id (mt/user->id :crowberto), :model "card", :model_id (u/id card), :context :question}@@ -237,7 +238,7 @@(mt/with-premium-features #{:audit-app}(testing "Running a query for a card in a dashboard is recorded in the view_log."(mt/with-temp [:model/Dashboard dash {}- :model/Card card {}+ :model/Card card {:dataset_query (mt/mbql-query venues {:limit 1})}:model/DashboardCard dashcard {:dashboard_id (:id dash):card_id (:id card)}](testing "POST /api/dashboard/:dashboard-id/card/:card-id/query"@@ -280,7 +281,8 @@(mt/with-premium-features #{:audit-app}(testing "Viewing an embedding logs the correct view log event."(embed-test/with-embedding-enabled-and-new-secret-key!- (mt/with-temp [:model/Card card {:enable_embedding true}]+ (mt/with-temp [:model/Card card {:enable_embedding true+ :dataset_query (mt/mbql-query venues {:limit 1})}](testing "GET /api/embed/card/:token/query"(client/client :get 202 (str (embed-test/card-url card) "/query"))(is (partial= {:model "card", :model_id (:id card), :has_access true}@@ -291,7 +293,7 @@(testing "Running a query for a card in a public dashboard logs the correct view log event."(embed-test/with-embedding-enabled-and-new-secret-key!(mt/with-temp [:model/Dashboard dash {:enable_embedding true}- :model/Card card {}+ :model/Card card {:dataset_query (mt/mbql-query venues {:limit 1})}:model/DashboardCard dashcard {:dashboard_id (:id dash):card_id (:id card)}](testing "GET /dashboard/:token/dashcard/:dashcard-id/card/:card-id"