Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Add more fields to default query remarks (#84744)

Riley Thompson1b1a733b188f
diff --git a/docs/developers-guide/driver-changelog.md b/docs/developers-guide/driver-changelog.md
index 8d0d6349332..a8028c0b361 100644
--- a/docs/developers-guide/driver-changelog.md
+++ b/docs/developers-guide/driver-changelog.md
@@ -28,6 +28,13 @@ title: Driver interface changelog
that matches no synced table or transform target. Dependency tracking recomputes such a query once sync finds the
table. Drivers built on `:sql` get this from `metabase.sql-tools.core/referenced-tables`.
+- `driver-api/default-query->remark` now includes the following fields when they are set: `userID`, `queryType`,
+ `queryHash`, `accountID`, `dashboardID`, `cardID`, `pulseID`, `context`, and `databaseID`. Drivers that override
+ `driver-api/query->remark` and add some of these fields to the default remark should remove them.
+
+- `driver-api/query->remark-data` has been added. It returns the data of the default query remark as a map, so that
+ drivers that format the remark in a different way (for example as JSON) include the same data.
+
## Metabase 0.64.0
- Date and time functions that depended on the `start-of-week` setting now take a `time-config` map as their first
diff --git a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj
index d0cb4d42517..719740a44ff 100644
--- a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj
@@ -38,6 +38,7 @@
[metabase.query-processor.util :as qp.util]
[metabase.query-processor.util.add-alias-info :as add]
[metabase.request.core :as request]
+ [metabase.system.core :as system]
[metabase.test :as mt]
[metabase.test.data :as data]
[metabase.test.data.interface :as tx]
@@ -647,7 +648,10 @@
(testing "make sure GTAP queries still include ID of user who ran them in the remark"
(met/with-gtaps! {:gtaps {:venues (venues-category-mbql-gtap-def)}
:attributes {"cat" 50}}
- (is (= (format "Metabase:: userID: %d queryType: MBQL queryHash: <hash>" (mt/user->id :rasta))
+ (is (= (format "Metabase:: userID: %d queryType: MBQL queryHash: <hash> accountID: %s context: ad-hoc databaseID: %d"
+ (mt/user->id :rasta)
+ (system/site-uuid)
+ (mt/id))
(run-query-returning-remark!
(fn []
(mt/user-http-request :rasta :post "dataset" (mt/mbql-query venues {:aggregation [[:count]]})))))))))
diff --git a/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/tests/utils.unit.spec.ts b/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/tests/utils.unit.spec.ts
index 01af90a32fd..f6b04f72aa6 100644
--- a/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/tests/utils.unit.spec.ts
+++ b/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/tests/utils.unit.spec.ts
@@ -5,6 +5,8 @@ describe("stripRemarks", () => {
Syntax error in SQL statement " FWEFWEF[*] "; expected "FROM, {"; SQL statement: -- Metabase:: userID: 1 queryType: native queryHash: 9863b8284f269ce8763ad59b04cec26407a1dd74eebeb16cffdf1ef3e23b325a\nfwefwef [42001-197]`;
const errorH2Stripped = `
Syntax error in SQL statement " FWEFWEF[*] "; expected "FROM, {"; SQL statement: fwefwef [42001-197]`;
+ const errorH2WithExtraFields = `
+ Syntax error in SQL statement " FWEFWEF[*] "; expected "FROM, {"; SQL statement: -- Metabase:: userID: 1 queryType: native queryHash: 9863b8284f269ce8763ad59b04cec26407a1dd74eebeb16cffdf1ef3e23b325a accountID: 1b4e9a2f-1c3d-4e5f-8a9b-0c1d2e3f4a5b cardID: 10 context: question databaseID: 1\nfwefwef [42001-197]`;
it("should strip remarks from query with stripRemarks", () => {
expect(stripRemarks(errorH2Unstripped)).toEqual(errorH2Stripped);
@@ -13,4 +15,8 @@ describe("stripRemarks", () => {
it("stripping stripped errors unchanged", () => {
expect(stripRemarks(errorH2Stripped)).toEqual(errorH2Stripped);
});
+
+ it("should strip remarks with extra fields", () => {
+ expect(stripRemarks(errorH2WithExtraFields)).toEqual(errorH2Stripped);
+ });
});
diff --git a/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/utils.ts b/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/utils.ts
index aa2dc000136..9630f8216ae 100644
--- a/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/utils.ts
+++ b/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/utils.ts
@@ -22,7 +22,7 @@ export function stripRemarks(error: string | undefined): string {
return "";
}
return error.replace(
- /-- Metabase:: userID: \d+ queryType: native queryHash: \w+\n/,
+ /-- Metabase:: userID: \d+ queryType: native queryHash: \w+[^\n]*\n/,
"",
);
}
diff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.clj
index e98bb6bc169..9251f256dbe 100644
--- a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.clj
+++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.clj
@@ -23,6 +23,7 @@
[metabase.query-processor.test :as qp]
[metabase.query-processor.util.add-alias-info :as add]
[metabase.sync.core :as sync]
+ [metabase.system.core :as system]
[metabase.test :as mt]
[metabase.test.data.bigquery-cloud-sdk :as bigquery.tx]
[metabase.test.data.impl :as data.impl]
@@ -257,7 +258,9 @@
" `v4_test_data.venues`.`price` AS `price` "
"FROM `v4_test_data.venues` "
"LIMIT 1"
- "\n\n-- Metabase:: userID: 1000 queryType: MBQL queryHash: 01020304"))
+ (format "\n\n-- Metabase:: userID: 1000 queryType: MBQL queryHash: 01020304 accountID: %s databaseID: %d"
+ (system/site-uuid)
+ (mt/id))))
(query->native
{:database (mt/id)
:type :query
diff --git a/modules/drivers/redshift/src/metabase/driver/redshift.clj b/modules/drivers/redshift/src/metabase/driver/redshift.clj
index ec4acc8e193..c6cdf779d83 100644
--- a/modules/drivers/redshift/src/metabase/driver/redshift.clj
+++ b/modules/drivers/redshift/src/metabase/driver/redshift.clj
@@ -797,15 +797,16 @@
user-parameters))
(defmethod driver-api/query->remark :redshift
- [_ {{:keys [executed-by card-id dashboard-id]} :info, :as query}]
- (str "/* partner: \"metabase\", "
- (json/encode {:dashboard_id dashboard-id
- :chart_id card-id
- :optional_user_id executed-by
- :optional_account_id (driver-api/site-uuid)
- :filter_values (field->parameter-value query)})
- " */ "
- (driver-api/default-query->remark query)))
+ [_ query]
+ (let [{:keys [user-id card-id dashboard-id account-id]} (driver-api/query->remark-data query)]
+ (str "/* partner: \"metabase\", "
+ (json/encode {:dashboard_id dashboard-id
+ :chart_id card-id
+ :optional_user_id user-id
+ :optional_account_id account-id
+ :filter_values (field->parameter-value query)})
+ " */ "
+ (driver-api/default-query->remark query))))
(defmethod sql-jdbc.execute/set-parameter [:redshift java.time.ZonedDateTime]
[driver ps i t]
diff --git a/modules/drivers/redshift/test/metabase/driver/redshift_test.clj b/modules/drivers/redshift/test/metabase/driver/redshift_test.clj
index 8f17374d769..2e4b2fdd5a2 100644
--- a/modules/drivers/redshift/test/metabase/driver/redshift_test.clj
+++ b/modules/drivers/redshift/test/metabase/driver/redshift_test.clj
@@ -89,7 +89,7 @@
(deftest remark-test
(testing "if I run a Redshift query, does it get a remark added to it?"
(mt/test-driver :redshift
- (let [expected (for [line ["-- /* partner: \"metabase\", {\"dashboard_id\":5678,\"chart_id\":1234,\"optional_user_id\":1000,\"optional_account_id\":\"{{site-uuid}}\",\"filter_values\":{\"id\":[\"1\",\"2\",\"3\"]}} */ Metabase:: userID: 1000 queryType: MBQL queryHash: cb83d4f6eedc250edb0f2c16f8d9a21e5d42f322ccece1494c8ef3d634581fe2"
+ (let [expected (for [line ["-- /* partner: \"metabase\", {\"dashboard_id\":5678,\"chart_id\":1234,\"optional_user_id\":1000,\"optional_account_id\":\"{{site-uuid}}\",\"filter_values\":{\"id\":[\"1\",\"2\",\"3\"]}} */ Metabase:: userID: 1000 queryType: MBQL queryHash: cb83d4f6eedc250edb0f2c16f8d9a21e5d42f322ccece1494c8ef3d634581fe2 accountID: {{site-uuid}} dashboardID: 5678 cardID: 1234 context: ad-hoc databaseID: {{database-id}}"
"SELECT"
" \"{{schema}}\".\"test_data_users\".\"id\" AS \"id\","
" \"{{schema}}\".\"test_data_users\".\"name\" AS \"name\","
@@ -110,6 +110,7 @@
" 2000"]]
(-> line
(str/replace #"\Q{{site-uuid}}\E" (system/site-uuid))
+ (str/replace #"\Q{{database-id}}\E" (str (mt/id)))
(str/replace #"\Q{{schema}}\E" (redshift.tx/unique-session-schema))))]
(is (= expected
(sql->lines
diff --git a/modules/drivers/snowflake/src/metabase/driver/snowflake.clj b/modules/drivers/snowflake/src/metabase/driver/snowflake.clj
index f3c3197461f..ccf6f3fe8ad 100644
--- a/modules/drivers/snowflake/src/metabase/driver/snowflake.clj
+++ b/modules/drivers/snowflake/src/metabase/driver/snowflake.clj
@@ -2,7 +2,6 @@
"Snowflake Driver."
(:refer-clojure :exclude [some select-keys not-empty mapv])
(:require
- [buddy.core.codecs :as codecs]
[clojure.java.jdbc :as jdbc]
[clojure.set :as set]
[clojure.string :as str]
@@ -1233,19 +1232,19 @@
(str sql "\n\n-- " remark))
(defmethod driver-api/query->remark :snowflake
- [_ {{:keys [context executed-by card-id pulse-id dashboard-id query-hash]} :info,
- query-type :type,
- database-id :database}]
- (json/encode {:client "Metabase"
- :context context
- :queryType query-type
- :userId executed-by
- :pulseId pulse-id
- :cardId card-id
- :dashboardId dashboard-id
- :databaseId database-id
- :queryHash (when (bytes? query-hash) (codecs/bytes->hex query-hash))
- :serverId (driver-api/site-uuid)}))
+ [_ {query-type :type, :as query}]
+ (let [{:keys [context user-id pulse-id card-id dashboard-id database-id query-hash account-id]}
+ (driver-api/query->remark-data query)]
+ (json/encode {:client "Metabase"
+ :context context
+ :queryType query-type
+ :userId user-id
+ :pulseId pulse-id
+ :cardId card-id
+ :dashboardId dashboard-id
+ :databaseId database-id
+ :queryHash query-hash
+ :serverId account-id})))
;;; ------------------------------------------------- User Impersonation --------------------------------------------------
diff --git a/modules/drivers/sparksql/test/metabase/driver/sparksql_test.clj b/modules/drivers/sparksql/test/metabase/driver/sparksql_test.clj
index 2da82f57b7e..2747d5662d9 100644
--- a/modules/drivers/sparksql/test/metabase/driver/sparksql_test.clj
+++ b/modules/drivers/sparksql/test/metabase/driver/sparksql_test.clj
@@ -12,6 +12,7 @@
[metabase.lib.test-util.notebook-helpers :as lib.tu.notebook]
[metabase.query-processor.compile :as qp.compile]
[metabase.query-processor.test :as qp]
+ [metabase.system.core :as system]
[metabase.test :as mt]))
(set! *warn-on-reflection* true)
@@ -89,7 +90,7 @@
(orig driver conn sql params)))]
(is (=? {:status :completed}
(qp/process-query query)))
- (is (= ["-- Metabase"
+ (is (= [(format "-- Metabase accountID: %s databaseID: %d" (system/site-uuid) (mt/id))
"SELECT"
" COUNT(*) AS `count`"
"FROM"
diff --git a/modules/drivers/starburst/src/metabase/driver/starburst.clj b/modules/drivers/starburst/src/metabase/driver/starburst.clj
index 894ce29380f..39161172f16 100644
--- a/modules/drivers/starburst/src/metabase/driver/starburst.clj
+++ b/modules/drivers/starburst/src/metabase/driver/starburst.clj
@@ -88,20 +88,6 @@
[_driver]
true)
-(defn- format-field
- [name value]
- (if (nil? value)
- ""
- (str " " name ": " value)))
-
-(defmethod driver-api/query->remark :starburst
- [_ {{:keys [card-id dashboard-id]} :info, :as query}]
- (str
- (driver-api/default-query->remark query)
- (format-field "accountID" (driver-api/site-uuid))
- (format-field "dashboardID" dashboard-id)
- (format-field "cardID" card-id)))
-
(defn- handle-execution-error-details
[^Exception e details]
(let [message (.getMessage e)
diff --git a/src/metabase/driver_api/core.clj b/src/metabase/driver_api/core.clj
index d57781c4f71..e2aac8e9386 100644
--- a/src/metabase/driver_api/core.clj
+++ b/src/metabase/driver_api/core.clj
@@ -169,6 +169,7 @@
qp.util.transformations.nest-breakouts/nest-pivot-joins
qp.util/default-query->remark
qp.util/query->remark
+ qp.util/query->remark-data
qp.wrap-value-literals/wrap-value-literals-in-mbql5
qp.writeback/execute-write-sql!
qp/process-query
diff --git a/src/metabase/query_processor/core.clj b/src/metabase/query_processor/core.clj
index 0813bb96f33..3908a81eb57 100644
--- a/src/metabase/query_processor/core.clj
+++ b/src/metabase/query_processor/core.clj
@@ -81,6 +81,7 @@
default-query->remark
internal-query?
query->remark
+ query->remark-data
query-hash
userland-query?]
;; Writeback — metabase.query-processor.writeback
diff --git a/src/metabase/query_processor/util.clj b/src/metabase/query_processor/util.clj
index ce1ed024adb..be74deccf22 100644
--- a/src/metabase/query_processor/util.clj
+++ b/src/metabase/query_processor/util.clj
@@ -8,6 +8,7 @@
[metabase.lib-be.core]
[metabase.lib.core :as lib]
[metabase.query-processor.schema :as qp.schema]
+ [metabase.system.core :as system]
[metabase.util :as u]
[metabase.util.malli :as mu]
[metabase.util.performance :refer [select-keys get-in]]
@@ -24,23 +25,47 @@
;; TODO - I think most of the functions in this namespace that we don't remove could be moved
;; to [[metabase.legacy-mbql.util]]
+(defn- remark-field
+ [field-name value]
+ (when-let [value (cond-> value (keyword? value) name)]
+ (format " %s: %s" field-name value)))
+
+(mu/defn query->remark-data
+ "Returns the data for a query remark as a map. Drivers can format this data as their database requires."
+ [{{:keys [executed-by query-hash card-id dashboard-id pulse-id context], :as _info} :info
+ database-id :database
+ :as query} :- ::qp.schema/any-query]
+ {:user-id executed-by
+ :query-type (if (:lib/type query)
+ (case (keyword (:lib/type (lib/query-stage query -1)))
+ :mbql.stage/mbql "MBQL"
+ :mbql.stage/native "native")
+ (case (keyword (:type query))
+ :query "MBQL"
+ :native "native"))
+ :query-hash (when (bytes? query-hash) (codecs/bytes->hex query-hash))
+ :account-id (system/site-uuid)
+ :dashboard-id dashboard-id
+ :card-id card-id
+ :pulse-id pulse-id
+ :context context
+ :database-id database-id})
+
(mu/defn default-query->remark
"Generates the default query remark. Exists as a separate function so that overrides of the query->remark multimethod
can access the default value."
- [{{:keys [executed-by query-hash], :as _info} :info, :as query} :- ::qp.schema/any-query]
- (let [query-type (if (:lib/type query)
- (case (keyword (:lib/type (lib/query-stage query -1)))
- :mbql.stage/mbql "MBQL"
- :mbql.stage/native "native")
- (case (keyword (:type query))
- :query "MBQL"
- :native "native"))]
- (str "Metabase" (when executed-by
- (assert (bytes? query-hash) "If info includes executed-by it should also include query-hash")
- (format ":: userID: %s queryType: %s queryHash: %s"
- executed-by
- query-type
- (codecs/bytes->hex query-hash))))))
+ [query :- ::qp.schema/any-query]
+ (let [{:keys [user-id query-type query-hash account-id dashboard-id card-id pulse-id context database-id]}
+ (query->remark-data query)]
+ (str "Metabase" (when user-id
+ (assert query-hash "If info includes executed-by it should also include query-hash")
+ (format ":: userID: %s queryType: %s queryHash: %s" user-id query-type query-hash))
+ (remark-field "accountID" account-id)
+ (remark-field "dashboardID" dashboard-id)
+ (remark-field "cardID" card-id)
+ (remark-field "pulseID" pulse-id)
+ (remark-field "context" context)
+ (remark-field "databaseID" database-id))))
(defmulti query->remark
"Generate an appropriate remark `^String` to be prepended to a query to give DBAs additional information about the query
diff --git a/test/metabase/driver/postgres_test.clj b/test/metabase/driver/postgres_test.clj
index 2ff9e924a99..97acbd9f068 100644
--- a/test/metabase/driver/postgres_test.clj
+++ b/test/metabase/driver/postgres_test.clj
@@ -57,6 +57,7 @@
[metabase.sync.sync-metadata :as sync-metadata]
[metabase.sync.sync-metadata.tables :as sync-tables]
[metabase.sync.util :as sync-util]
+ [metabase.system.core :as system]
[metabase.test :as mt]
[metabase.test.data.interface :as tx]
[metabase.util :as u]
@@ -1612,7 +1613,11 @@
:type :native
:native {:query "SELECT adsasdasd;"}})
(catch Throwable e
- (is (= "ERROR: column \"adsasdasd\" does not exist\n Position: 20"
+ ;; The position counts the `-- <remark>\n` line that comes before the query.
+ (is (= (format "ERROR: column \"adsasdasd\" does not exist\n Position: %d"
+ (+ (count (format "-- Metabase accountID: %s databaseID: %d\n" (system/site-uuid) (mt/id)))
+ (count "SELECT ")
+ 1))
(try
(-> e ex-cause ex-message)
(catch Throwable e
diff --git a/test/metabase/query_processor/util_test.clj b/test/metabase/query_processor/util_test.clj
new file mode 100644
index 00000000000..e5c90f2cfb9
--- /dev/null
+++ b/test/metabase/query_processor/util_test.clj
@@ -0,0 +1,37 @@
+(ns metabase.query-processor.util-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.lib.core :as lib]
+ [metabase.lib.test-metadata :as meta]
+ [metabase.query-processor.util :as qp.util]
+ [metabase.system.core :as system]))
+
+(defn- venues-query []
+ (lib/query meta/metadata-provider (meta/table-metadata :venues)))
+
+(deftest ^:parallel default-query->remark-test
+ (testing "Without query info, the remark has the instance and database IDs only"
+ (is (= (format "Metabase accountID: %s databaseID: %d" (system/site-uuid) (meta/id))
+ (qp.util/default-query->remark (venues-query)))))
+ (testing "With query info, the remark has all the fields that are set"
+ (is (= (format (str "Metabase:: userID: 1000 queryType: MBQL queryHash: 01020304"
+ " accountID: %s dashboardID: 20 cardID: 10 pulseID: 30 context: pulse databaseID: %d")
+ (system/site-uuid)
+ (meta/id))
+ (qp.util/default-query->remark
+ (assoc (venues-query) :info {:executed-by 1000
+ :query-hash (byte-array [1 2 3 4])
+ :card-id 10
+ :dashboard-id 20
+ :pulse-id 30
+ :context :pulse})))))
+ (testing "Native queries have the native query type"
+ (is (= (format "Metabase:: userID: 1000 queryType: native queryHash: 01020304 accountID: %s cardID: 10 context: question databaseID: %d"
+ (system/site-uuid)
+ (meta/id))
+ (qp.util/default-query->remark
+ (assoc (lib/native-query meta/metadata-provider "SELECT 1")
+ :info {:executed-by 1000
+ :query-hash (byte-array [1 2 3 4])
+ :card-id 10
+ :context :question}))))))