The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Add more fields to default query remarks (#84744)
Riley Thompson1b1a733b188f
diff --git a/docs/developers-guide/driver-changelog.md b/docs/developers-guide/driver-changelog.mdindex 8d0d6349332..a8028c0b361 100644--- a/docs/developers-guide/driver-changelog.md+++ b/docs/developers-guide/driver-changelog.md@@ -28,6 +28,13 @@ title: Driver interface changelogthat matches no synced table or transform target. Dependency tracking recomputes such a query once sync finds thetable. Drivers built on `:sql` get this from `metabase.sql-tools.core/referenced-tables`.+- `driver-api/default-query->remark` now includes the following fields when they are set: `userID`, `queryType`,+ `queryHash`, `accountID`, `dashboardID`, `cardID`, `pulseID`, `context`, and `databaseID`. Drivers that override+ `driver-api/query->remark` and add some of these fields to the default remark should remove them.++- `driver-api/query->remark-data` has been added. It returns the data of the default query remark as a map, so that+ drivers that format the remark in a different way (for example as JSON) include the same data.+## Metabase 0.64.0- Date and time functions that depended on the `start-of-week` setting now take a `time-config` map as their firstdiff --git a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.cljindex d0cb4d42517..719740a44ff 100644--- a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj+++ b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj@@ -38,6 +38,7 @@[metabase.query-processor.util :as qp.util][metabase.query-processor.util.add-alias-info :as add][metabase.request.core :as request]+ [metabase.system.core :as system][metabase.test :as mt][metabase.test.data :as data][metabase.test.data.interface :as tx]@@ -647,7 +648,10 @@(testing "make sure GTAP queries still include ID of user who ran them in the remark"(met/with-gtaps! {:gtaps {:venues (venues-category-mbql-gtap-def)}:attributes {"cat" 50}}- (is (= (format "Metabase:: userID: %d queryType: MBQL queryHash: <hash>" (mt/user->id :rasta))+ (is (= (format "Metabase:: userID: %d queryType: MBQL queryHash: <hash> accountID: %s context: ad-hoc databaseID: %d"+ (mt/user->id :rasta)+ (system/site-uuid)+ (mt/id))(run-query-returning-remark!(fn [](mt/user-http-request :rasta :post "dataset" (mt/mbql-query venues {:aggregation [[:count]]})))))))))diff --git a/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/tests/utils.unit.spec.ts b/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/tests/utils.unit.spec.tsindex 01af90a32fd..f6b04f72aa6 100644--- a/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/tests/utils.unit.spec.ts+++ b/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/tests/utils.unit.spec.ts@@ -5,6 +5,8 @@ describe("stripRemarks", () => {Syntax error in SQL statement " FWEFWEF[*] "; expected "FROM, {"; SQL statement: -- Metabase:: userID: 1 queryType: native queryHash: 9863b8284f269ce8763ad59b04cec26407a1dd74eebeb16cffdf1ef3e23b325a\nfwefwef [42001-197]`;const errorH2Stripped = `Syntax error in SQL statement " FWEFWEF[*] "; expected "FROM, {"; SQL statement: fwefwef [42001-197]`;+ const errorH2WithExtraFields = `+ Syntax error in SQL statement " FWEFWEF[*] "; expected "FROM, {"; SQL statement: -- Metabase:: userID: 1 queryType: native queryHash: 9863b8284f269ce8763ad59b04cec26407a1dd74eebeb16cffdf1ef3e23b325a accountID: 1b4e9a2f-1c3d-4e5f-8a9b-0c1d2e3f4a5b cardID: 10 context: question databaseID: 1\nfwefwef [42001-197]`;it("should strip remarks from query with stripRemarks", () => {expect(stripRemarks(errorH2Unstripped)).toEqual(errorH2Stripped);@@ -13,4 +15,8 @@ describe("stripRemarks", () => {it("stripping stripped errors unchanged", () => {expect(stripRemarks(errorH2Stripped)).toEqual(errorH2Stripped);});++ it("should strip remarks with extra fields", () => {+ expect(stripRemarks(errorH2WithExtraFields)).toEqual(errorH2Stripped);+ });});diff --git a/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/utils.ts b/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/utils.tsindex aa2dc000136..9630f8216ae 100644--- a/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/utils.ts+++ b/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/utils.ts@@ -22,7 +22,7 @@ export function stripRemarks(error: string | undefined): string {return "";}return error.replace(- /-- Metabase:: userID: \d+ queryType: native queryHash: \w+\n/,+ /-- Metabase:: userID: \d+ queryType: native queryHash: \w+[^\n]*\n/,"",);}diff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.cljindex e98bb6bc169..9251f256dbe 100644--- a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.clj+++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.clj@@ -23,6 +23,7 @@[metabase.query-processor.test :as qp][metabase.query-processor.util.add-alias-info :as add][metabase.sync.core :as sync]+ [metabase.system.core :as system][metabase.test :as mt][metabase.test.data.bigquery-cloud-sdk :as bigquery.tx][metabase.test.data.impl :as data.impl]@@ -257,7 +258,9 @@" `v4_test_data.venues`.`price` AS `price` ""FROM `v4_test_data.venues` ""LIMIT 1"- "\n\n-- Metabase:: userID: 1000 queryType: MBQL queryHash: 01020304"))+ (format "\n\n-- Metabase:: userID: 1000 queryType: MBQL queryHash: 01020304 accountID: %s databaseID: %d"+ (system/site-uuid)+ (mt/id))))(query->native{:database (mt/id):type :querydiff --git a/modules/drivers/redshift/src/metabase/driver/redshift.clj b/modules/drivers/redshift/src/metabase/driver/redshift.cljindex ec4acc8e193..c6cdf779d83 100644--- a/modules/drivers/redshift/src/metabase/driver/redshift.clj+++ b/modules/drivers/redshift/src/metabase/driver/redshift.clj@@ -797,15 +797,16 @@user-parameters))(defmethod driver-api/query->remark :redshift- [_ {{:keys [executed-by card-id dashboard-id]} :info, :as query}]- (str "/* partner: \"metabase\", "- (json/encode {:dashboard_id dashboard-id- :chart_id card-id- :optional_user_id executed-by- :optional_account_id (driver-api/site-uuid)- :filter_values (field->parameter-value query)})- " */ "- (driver-api/default-query->remark query)))+ [_ query]+ (let [{:keys [user-id card-id dashboard-id account-id]} (driver-api/query->remark-data query)]+ (str "/* partner: \"metabase\", "+ (json/encode {:dashboard_id dashboard-id+ :chart_id card-id+ :optional_user_id user-id+ :optional_account_id account-id+ :filter_values (field->parameter-value query)})+ " */ "+ (driver-api/default-query->remark query))))(defmethod sql-jdbc.execute/set-parameter [:redshift java.time.ZonedDateTime][driver ps i t]diff --git a/modules/drivers/redshift/test/metabase/driver/redshift_test.clj b/modules/drivers/redshift/test/metabase/driver/redshift_test.cljindex 8f17374d769..2e4b2fdd5a2 100644--- a/modules/drivers/redshift/test/metabase/driver/redshift_test.clj+++ b/modules/drivers/redshift/test/metabase/driver/redshift_test.clj@@ -89,7 +89,7 @@(deftest remark-test(testing "if I run a Redshift query, does it get a remark added to it?"(mt/test-driver :redshift- (let [expected (for [line ["-- /* partner: \"metabase\", {\"dashboard_id\":5678,\"chart_id\":1234,\"optional_user_id\":1000,\"optional_account_id\":\"{{site-uuid}}\",\"filter_values\":{\"id\":[\"1\",\"2\",\"3\"]}} */ Metabase:: userID: 1000 queryType: MBQL queryHash: cb83d4f6eedc250edb0f2c16f8d9a21e5d42f322ccece1494c8ef3d634581fe2"+ (let [expected (for [line ["-- /* partner: \"metabase\", {\"dashboard_id\":5678,\"chart_id\":1234,\"optional_user_id\":1000,\"optional_account_id\":\"{{site-uuid}}\",\"filter_values\":{\"id\":[\"1\",\"2\",\"3\"]}} */ Metabase:: userID: 1000 queryType: MBQL queryHash: cb83d4f6eedc250edb0f2c16f8d9a21e5d42f322ccece1494c8ef3d634581fe2 accountID: {{site-uuid}} dashboardID: 5678 cardID: 1234 context: ad-hoc databaseID: {{database-id}}""SELECT"" \"{{schema}}\".\"test_data_users\".\"id\" AS \"id\","" \"{{schema}}\".\"test_data_users\".\"name\" AS \"name\","@@ -110,6 +110,7 @@" 2000"]](-> line(str/replace #"\Q{{site-uuid}}\E" (system/site-uuid))+ (str/replace #"\Q{{database-id}}\E" (str (mt/id)))(str/replace #"\Q{{schema}}\E" (redshift.tx/unique-session-schema))))](is (= expected(sql->linesdiff --git a/modules/drivers/snowflake/src/metabase/driver/snowflake.clj b/modules/drivers/snowflake/src/metabase/driver/snowflake.cljindex f3c3197461f..ccf6f3fe8ad 100644--- a/modules/drivers/snowflake/src/metabase/driver/snowflake.clj+++ b/modules/drivers/snowflake/src/metabase/driver/snowflake.clj@@ -2,7 +2,6 @@"Snowflake Driver."(:refer-clojure :exclude [some select-keys not-empty mapv])(:require- [buddy.core.codecs :as codecs][clojure.java.jdbc :as jdbc][clojure.set :as set][clojure.string :as str]@@ -1233,19 +1232,19 @@(str sql "\n\n-- " remark))(defmethod driver-api/query->remark :snowflake- [_ {{:keys [context executed-by card-id pulse-id dashboard-id query-hash]} :info,- query-type :type,- database-id :database}]- (json/encode {:client "Metabase"- :context context- :queryType query-type- :userId executed-by- :pulseId pulse-id- :cardId card-id- :dashboardId dashboard-id- :databaseId database-id- :queryHash (when (bytes? query-hash) (codecs/bytes->hex query-hash))- :serverId (driver-api/site-uuid)}))+ [_ {query-type :type, :as query}]+ (let [{:keys [context user-id pulse-id card-id dashboard-id database-id query-hash account-id]}+ (driver-api/query->remark-data query)]+ (json/encode {:client "Metabase"+ :context context+ :queryType query-type+ :userId user-id+ :pulseId pulse-id+ :cardId card-id+ :dashboardId dashboard-id+ :databaseId database-id+ :queryHash query-hash+ :serverId account-id})));;; ------------------------------------------------- User Impersonation --------------------------------------------------diff --git a/modules/drivers/sparksql/test/metabase/driver/sparksql_test.clj b/modules/drivers/sparksql/test/metabase/driver/sparksql_test.cljindex 2da82f57b7e..2747d5662d9 100644--- a/modules/drivers/sparksql/test/metabase/driver/sparksql_test.clj+++ b/modules/drivers/sparksql/test/metabase/driver/sparksql_test.clj@@ -12,6 +12,7 @@[metabase.lib.test-util.notebook-helpers :as lib.tu.notebook][metabase.query-processor.compile :as qp.compile][metabase.query-processor.test :as qp]+ [metabase.system.core :as system][metabase.test :as mt]))(set! *warn-on-reflection* true)@@ -89,7 +90,7 @@(orig driver conn sql params)))](is (=? {:status :completed}(qp/process-query query)))- (is (= ["-- Metabase"+ (is (= [(format "-- Metabase accountID: %s databaseID: %d" (system/site-uuid) (mt/id))"SELECT"" COUNT(*) AS `count`""FROM"diff --git a/modules/drivers/starburst/src/metabase/driver/starburst.clj b/modules/drivers/starburst/src/metabase/driver/starburst.cljindex 894ce29380f..39161172f16 100644--- a/modules/drivers/starburst/src/metabase/driver/starburst.clj+++ b/modules/drivers/starburst/src/metabase/driver/starburst.clj@@ -88,20 +88,6 @@[_driver]true)-(defn- format-field- [name value]- (if (nil? value)- ""- (str " " name ": " value)))--(defmethod driver-api/query->remark :starburst- [_ {{:keys [card-id dashboard-id]} :info, :as query}]- (str- (driver-api/default-query->remark query)- (format-field "accountID" (driver-api/site-uuid))- (format-field "dashboardID" dashboard-id)- (format-field "cardID" card-id)))-(defn- handle-execution-error-details[^Exception e details](let [message (.getMessage e)diff --git a/src/metabase/driver_api/core.clj b/src/metabase/driver_api/core.cljindex d57781c4f71..e2aac8e9386 100644--- a/src/metabase/driver_api/core.clj+++ b/src/metabase/driver_api/core.clj@@ -169,6 +169,7 @@qp.util.transformations.nest-breakouts/nest-pivot-joinsqp.util/default-query->remarkqp.util/query->remark+ qp.util/query->remark-dataqp.wrap-value-literals/wrap-value-literals-in-mbql5qp.writeback/execute-write-sql!qp/process-querydiff --git a/src/metabase/query_processor/core.clj b/src/metabase/query_processor/core.cljindex 0813bb96f33..3908a81eb57 100644--- a/src/metabase/query_processor/core.clj+++ b/src/metabase/query_processor/core.clj@@ -81,6 +81,7 @@default-query->remarkinternal-query?query->remark+ query->remark-dataquery-hashuserland-query?];; Writeback — metabase.query-processor.writebackdiff --git a/src/metabase/query_processor/util.clj b/src/metabase/query_processor/util.cljindex ce1ed024adb..be74deccf22 100644--- a/src/metabase/query_processor/util.clj+++ b/src/metabase/query_processor/util.clj@@ -8,6 +8,7 @@[metabase.lib-be.core][metabase.lib.core :as lib][metabase.query-processor.schema :as qp.schema]+ [metabase.system.core :as system][metabase.util :as u][metabase.util.malli :as mu][metabase.util.performance :refer [select-keys get-in]]@@ -24,23 +25,47 @@;; TODO - I think most of the functions in this namespace that we don't remove could be moved;; to [[metabase.legacy-mbql.util]]+(defn- remark-field+ [field-name value]+ (when-let [value (cond-> value (keyword? value) name)]+ (format " %s: %s" field-name value)))++(mu/defn query->remark-data+ "Returns the data for a query remark as a map. Drivers can format this data as their database requires."+ [{{:keys [executed-by query-hash card-id dashboard-id pulse-id context], :as _info} :info+ database-id :database+ :as query} :- ::qp.schema/any-query]+ {:user-id executed-by+ :query-type (if (:lib/type query)+ (case (keyword (:lib/type (lib/query-stage query -1)))+ :mbql.stage/mbql "MBQL"+ :mbql.stage/native "native")+ (case (keyword (:type query))+ :query "MBQL"+ :native "native"))+ :query-hash (when (bytes? query-hash) (codecs/bytes->hex query-hash))+ :account-id (system/site-uuid)+ :dashboard-id dashboard-id+ :card-id card-id+ :pulse-id pulse-id+ :context context+ :database-id database-id})+(mu/defn default-query->remark"Generates the default query remark. Exists as a separate function so that overrides of the query->remark multimethodcan access the default value."- [{{:keys [executed-by query-hash], :as _info} :info, :as query} :- ::qp.schema/any-query]- (let [query-type (if (:lib/type query)- (case (keyword (:lib/type (lib/query-stage query -1)))- :mbql.stage/mbql "MBQL"- :mbql.stage/native "native")- (case (keyword (:type query))- :query "MBQL"- :native "native"))]- (str "Metabase" (when executed-by- (assert (bytes? query-hash) "If info includes executed-by it should also include query-hash")- (format ":: userID: %s queryType: %s queryHash: %s"- executed-by- query-type- (codecs/bytes->hex query-hash))))))+ [query :- ::qp.schema/any-query]+ (let [{:keys [user-id query-type query-hash account-id dashboard-id card-id pulse-id context database-id]}+ (query->remark-data query)]+ (str "Metabase" (when user-id+ (assert query-hash "If info includes executed-by it should also include query-hash")+ (format ":: userID: %s queryType: %s queryHash: %s" user-id query-type query-hash))+ (remark-field "accountID" account-id)+ (remark-field "dashboardID" dashboard-id)+ (remark-field "cardID" card-id)+ (remark-field "pulseID" pulse-id)+ (remark-field "context" context)+ (remark-field "databaseID" database-id))))(defmulti query->remark"Generate an appropriate remark `^String` to be prepended to a query to give DBAs additional information about the querydiff --git a/test/metabase/driver/postgres_test.clj b/test/metabase/driver/postgres_test.cljindex 2ff9e924a99..97acbd9f068 100644--- a/test/metabase/driver/postgres_test.clj+++ b/test/metabase/driver/postgres_test.clj@@ -57,6 +57,7 @@[metabase.sync.sync-metadata :as sync-metadata][metabase.sync.sync-metadata.tables :as sync-tables][metabase.sync.util :as sync-util]+ [metabase.system.core :as system][metabase.test :as mt][metabase.test.data.interface :as tx][metabase.util :as u]@@ -1612,7 +1613,11 @@:type :native:native {:query "SELECT adsasdasd;"}})(catch Throwable e- (is (= "ERROR: column \"adsasdasd\" does not exist\n Position: 20"+ ;; The position counts the `-- <remark>\n` line that comes before the query.+ (is (= (format "ERROR: column \"adsasdasd\" does not exist\n Position: %d"+ (+ (count (format "-- Metabase accountID: %s databaseID: %d\n" (system/site-uuid) (mt/id)))+ (count "SELECT ")+ 1))(try(-> e ex-cause ex-message)(catch Throwable ediff --git a/test/metabase/query_processor/util_test.clj b/test/metabase/query_processor/util_test.cljnew file mode 100644index 00000000000..e5c90f2cfb9--- /dev/null+++ b/test/metabase/query_processor/util_test.clj@@ -0,0 +1,37 @@+(ns metabase.query-processor.util-test+ (:require+ [clojure.test :refer :all]+ [metabase.lib.core :as lib]+ [metabase.lib.test-metadata :as meta]+ [metabase.query-processor.util :as qp.util]+ [metabase.system.core :as system]))++(defn- venues-query []+ (lib/query meta/metadata-provider (meta/table-metadata :venues)))++(deftest ^:parallel default-query->remark-test+ (testing "Without query info, the remark has the instance and database IDs only"+ (is (= (format "Metabase accountID: %s databaseID: %d" (system/site-uuid) (meta/id))+ (qp.util/default-query->remark (venues-query)))))+ (testing "With query info, the remark has all the fields that are set"+ (is (= (format (str "Metabase:: userID: 1000 queryType: MBQL queryHash: 01020304"+ " accountID: %s dashboardID: 20 cardID: 10 pulseID: 30 context: pulse databaseID: %d")+ (system/site-uuid)+ (meta/id))+ (qp.util/default-query->remark+ (assoc (venues-query) :info {:executed-by 1000+ :query-hash (byte-array [1 2 3 4])+ :card-id 10+ :dashboard-id 20+ :pulse-id 30+ :context :pulse})))))+ (testing "Native queries have the native query type"+ (is (= (format "Metabase:: userID: 1000 queryType: native queryHash: 01020304 accountID: %s cardID: 10 context: question databaseID: %d"+ (system/site-uuid)+ (meta/id))+ (qp.util/default-query->remark+ (assoc (lib/native-query meta/metadata-provider "SELECT 1")+ :info {:executed-by 1000+ :query-hash (byte-array [1 2 3 4])+ :card-id 10+ :context :question}))))))