Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

[perf] Tools to repro query + metadata locally, given instance API access (#67952)

Braden Shepherdsonb01732fd6904
diff --git a/dev/src/dev/debug_qp/analyze_user_query.clj b/dev/src/dev/debug_qp/analyze_user_query.clj
new file mode 100644
index 00000000000..e322a7e5414
--- /dev/null
+++ b/dev/src/dev/debug_qp/analyze_user_query.clj
@@ -0,0 +1,157 @@
+(ns dev.debug-qp.analyze-user-query
+ "Given access to e.g. a Cloud instance with a querying issue, this script can use a login token borrowed from your
+ browser to make API calls to the instance, and assemble a local mock `MetadataProvider` which contains the tables,
+ fields and cards necessary to run QP preprocessing and compilation locally against the same metadata the user
+ instance is seeing.
+
+ Of course the queries can't be executed without a connection to the user's DWH, which is usually impossible."
+ (:require
+ [clj-http.client :as http]
+ [medley.core :as m]
+ [metabase.lib.js.metadata :as lib.js.metadata]
+ [metabase.lib.metadata :as lib.metadata]
+ [metabase.query-processor.compile :as qp.compile]
+ [metabase.query-processor.preprocess :as qp.preprocess]
+ [metabase.util.json :as json]
+ [metabase.util.malli :as mu]))
+
+;; XXX: INSTRUCTIONS:
+;; - Log into the instance, open DevTools, and navigate to the problem question.
+;; - Open the Network tab, find a `/api/card/1234` or similar request, right click and "Copy as cURL".
+;; - Paste that somewhere, probably into your shell to test that the `curl` call is working.
+;; - Grab its `cookie` and edit it into `my-ctx` below. (It's the `-b` arg to curl.)
+;; - Edit the `instance-url` of `my-ctx` below likewise.
+;; - Edit the `database-id` of `my-ctx` below to match the problem database on the user instance.
+;; - That can be seen in eg. `/api/card` requests and similar.
+;; - Now you can run the analysis functions below, like compile-card.
+;; - See the examples at the bottom of this file.
+(def ^:private my-ctx
+ "Contains instance details, for passing to the various functions below."
+ {:instance-url "https://acustomer.metabaseapp.com" ; No trailing slash.
+ :cookie-from-browser "COPY ME FROM YOUR curl COMMAND"
+ :database-id 123})
+
+(defn- mk-headers [{:keys [cookie-from-browser instance-url] :as _ctx}]
+ {"accept" "application/json"
+ "accept-language" "en-US,en;q=0.9"
+ "cache-control" "no-cache"
+ "content-type" "application/json"
+ "cookie" cookie-from-browser
+ "pragma" "no-cache"
+ "priority" "u=1, i"
+ "referer" instance-url
+ "sec-ch-ua" "\"Google Chrome\";v=\"141\", \"Not?A_Brand\";v=\"8\", \"Chromium\";v=\"141\""
+ "sec-ch-ua-mobile" "?0"
+ "sec-ch-ua-platform" "\"macOS\""
+ "sec-fetch-dest" "empty"
+ "sec-fetch-mode" "cors"
+ "sec-fetch-site" "same-origin"
+ "user-agent" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36"})
+
+(defn- instance-fetch [{:keys [instance-url] :as ctx} path]
+ (let [headers (mk-headers ctx)
+ request {:method :get
+ :url (str instance-url path)
+ :accept :json
+ :content-type :json
+ :throw-exceptions false
+ :headers headers}
+ response (-> (http/request request)
+ (select-keys [:body :headers :status])
+ (update :body json/decode))]
+ (if (<= 200 (:status response) 299)
+ (:body response)
+ (throw (ex-info "Non-200 response" response)))))
+
+(defn- qm-index [objs]
+ (m/index-by (comp str #(get % "id")) objs))
+
+(defn- qm->metadata [raw]
+ ;; /query_metadata requests return a map in a different shape from what lib.js.metadata expects.
+ ;; We get {"databases" [{...}], "tables" [{..., "fields": [{...}]}]} but we need
+ ;; {"databases" {"12" {...}}
+ ;; "tables" {"333" {...}}
+ ;; "fields" {"4343" {...}}
+ ;; "questions" {"12" {...}}}
+ (let [fields (for [table (get raw "tables")
+ :when (number? (get table "id"))
+ field (get table "fields")
+ :when (number? (get field "id"))]
+ field)]
+ {"databases" (qm-index (get raw "databases"))
+ "tables" (qm-index (get raw "tables"))
+ "fields" (qm-index fields)}))
+
+(defn- recursive-card-metadata
+ "Helper for [[metadata-for]]. Prefer calling that function instead."
+ [ctx main-card-id]
+ (loop [metadata {}
+ cards-seen #{main-card-id}
+ [card-id & cards] [main-card-id]]
+ (if-not card-id
+ metadata
+ (let [card (instance-fetch ctx (str "/api/card/" card-id))
+ raw (instance-fetch ctx (str "/api/card/" card-id "/query_metadata"))
+ nested-cards (->> (get raw "tables")
+ (map #(get % "id"))
+ (filter string?)
+ (map (comp Long/parseLong #(subs % 6)))
+ (remove cards-seen))]
+ (recur (m/deep-merge metadata
+ {"questions" {(str card-id) card}}
+ (qm->metadata raw))
+ (into cards-seen nested-cards)
+ (concat cards nested-cards))))))
+
+(defn- table-metadata
+ "Helper for [[metadata-for]]. Prefer calling that function instead."
+ [ctx table-id]
+ (qm->metadata {"tables" [(-> (instance-fetch ctx (str "/api/table/" table-id "/query_metadata"))
+ (dissoc "db"))]}))
+
+;; =============================================== External API ===================================================
+(defn metadata-for
+ "Given a map like `{:card [123 456], :table [789]}` returns a `MetadataProvider` with metadata for all the tables,
+ cards and fields transitively required by the specified roots.
+
+ Generally you only need a single `{:card [123]}` for the problem card you're investigating.
+
+ Occasionally a broken card might reference fields by ID which are not really part of their query - then they're
+ missing from the `/query_metadata` on that card and will be missed by this function unless you explicitly add
+ `{:table [789]}` to the map of roots.
+
+ Returns a complete `MetadataProvider` ready to use in a QP call."
+ [{:keys [database-id] :as ctx} kind->ids]
+ (->> (for [[kind ids] kind->ids
+ id ids]
+ (case kind
+ :card (recursive-card-metadata ctx id)
+ :table (table-metadata ctx id)))
+ (apply m/deep-merge)
+ (lib.js.metadata/metadata-provider database-id)))
+
+(defn preprocess-card
+ "Given a `MetadataProvider` built by [[metadata-for]] and a card ID, tries to preprocess that card."
+ [metadata-provider card-id]
+ (mu/disable-enforcement
+ (-> (lib.metadata/card metadata-provider card-id)
+ :dataset-query
+ qp.preprocess/preprocess)))
+
+(defn compile-card
+ "Given a `MetadataProvider` built by [[metadata-for]] and a card ID, tries to compile that card to e.g. SQL."
+ [metadata-provider card-id]
+ (mu/disable-enforcement
+ (-> (lib.metadata/card metadata-provider card-id)
+ :dataset-query
+ qp.compile/compile)))
+
+(comment
+ ;; Example calls
+ (let [card-id 456
+ mp (metadata-for my-ctx {:card [card-id]})]
+ (compile-card mp card-id))
+
+ (let [card-id 456
+ mp (metadata-for my-ctx {:card [card-id]})]
+ (preprocess-card mp card-id)))
diff --git a/src/metabase/lib/js/metadata.cljs b/src/metabase/lib/js/metadata.cljc
similarity index 82%
rename from src/metabase/lib/js/metadata.cljs
rename to src/metabase/lib/js/metadata.cljc
index b44b4ee2b6b..bfc6cf64a4f 100644
--- a/src/metabase/lib/js/metadata.cljs
+++ b/src/metabase/lib/js/metadata.cljc
@@ -1,12 +1,14 @@
(ns metabase.lib.js.metadata
(:refer-clojure :exclude [keywordize-keys])
(:require
+ #?@(:clj (#_{:clj-kondo/ignore [:discouraged-namespace]}
+ [metabase.legacy-mbql.normalize :as legacy-mbql.normalize])
+ :cljs ([goog]
+ [goog.object :as gobject]
+ [metabase.lib.cache :as lib.cache]))
[clojure.core.protocols]
[clojure.string :as str]
- [goog]
- [goog.object :as gobject]
[medley.core :as m]
- [metabase.lib.cache :as lib.cache]
[metabase.lib.metadata.cached-provider :as lib.metadata.cached-provider]
[metabase.lib.metadata.protocols :as lib.metadata.protocols]
[metabase.lib.normalize :as lib.normalize]
@@ -29,8 +31,10 @@
;;; where keys are a map of String ID => metadata
(defn- object-get [obj k]
- (when (and obj (js-in k obj))
- (gobject/get obj k)))
+ #?(:cljs (when (and obj (js-in k obj))
+ (gobject/get obj k))
+ :clj (when (and obj (contains? obj k))
+ (get obj k))))
(defn- obj->clj
"Convert a JS object of *any* class to a ClojureScript object."
@@ -43,23 +47,26 @@
;; already a ClojureScript object.
(into {} xform (m/remove-keys skip-keys obj))
;; has a plain-JavaScript `_plainObject` attached: apply `xform` to it and call it a day
- (if-let [plain-object (when use-plain-object?
- (some-> (object-get obj "_plainObject")
- js->clj
- not-empty))]
- (into {} xform (m/remove-keys skip-keys plain-object))
- ;; otherwise do things the hard way and convert an arbitrary object into a Cljs map. (`js->clj` doesn't work on
- ;; arbitrary classes other than `Object`)
- (into {}
- (comp
- (remove (set skip-keys))
- (map (fn [k]
- [k (object-get obj k)]))
- ;; ignore values that are functions
- (remove (fn [[_k v]]
- (js-fn? v)))
- xform)
- (js-keys obj))))))
+ #?(:cljs (if-let [plain-object (when use-plain-object?
+ (some-> (object-get obj "_plainObject")
+ js->clj
+ not-empty))]
+ (into {} xform (m/remove-keys skip-keys plain-object))
+ ;; otherwise do things the hard way and convert an arbitrary object into a Cljs map. (`js->clj` doesn't work on
+ ;; arbitrary classes other than `Object`)
+ (into {}
+ (comp
+ (remove (set skip-keys))
+ (map (fn [k]
+ [k (object-get obj k)]))
+ ;; ignore values that are functions
+ (remove (fn [[_k v]]
+ (js-fn? v)))
+ xform)
+ (js-keys obj)))
+ :clj (when obj
+ (comment use-plain-object?) ; Touch this binding to make linting happy.
+ (throw (ex-info "_plainObject classy nesting not supported in CLJ" {:obj obj})))))))
;;; this intentionally does not use the lib hierarchy since it's not dealing with MBQL/lib keys
(defmulti ^:private excluded-keys
@@ -96,6 +103,7 @@
Use [[excluded-keys]] to drop keys from the input.
Defaults to nil, which means no renaming is done."
+ {:arglists '([object-type])}
identity)
(defmethod rename-key-fn :default [_]
@@ -112,7 +120,7 @@
rename-key (#(or (rename-key %) %)))
v]))
;; remove [[excluded-keys]]
- (if (empty? excluded-keys-set)
+ (if (perf/empty? excluded-keys-set)
identity
(remove (fn [[k _v]]
(contains? excluded-keys-set k))))
@@ -141,7 +149,7 @@
(let [parsed (assoc (obj->clj xform object opts) :lib/type lib-type-name)]
(log/debugf "Parsed metadata %s %s\n%s" object-type (:id parsed) (u/pprint-to-str parsed))
parsed)
- (catch js/Error e
+ (catch #?(:cljs js/Error :clj Exception) e
(log/errorf e "Error parsing %s %s: %s" object-type (pr-str object) (ex-message e))
nil)))))
@@ -175,10 +183,13 @@
[_object-type]
(fn [k v]
(case k
- :dbms-version (js->clj v :keywordize-keys true)
+ :dbms-version #?(:cljs (js->clj v :keywordize-keys true)
+ :clj (perf/keywordize-keys v))
+ :engine (keyword v)
:features (into #{} (map keyword) v)
:native-permissions (keyword v)
- :settings (js->clj v :keywordize-keys true)
+ :settings #?(:cljs (js->clj v :keywordize-keys true)
+ :clj (perf/keywordize-keys v))
v)))
(defmethod parse-objects-default-key :database
@@ -255,8 +266,8 @@
(defn- parse-field-values [field-values]
(when (= (object-get field-values "type") "full")
- {:values (js->clj (object-get field-values "values"))
- :human-readable-values (js->clj (object-get field-values "human_readable_values"))}))
+ {:values (#?(:cljs js->clj :clj identity) (object-get field-values "values"))
+ :human-readable-values (#?(:cljs js->clj :clj identity) (object-get field-values "human_readable_values"))}))
(defn- parse-dimension
"`:dimensions` comes in as an array for historical reasons, even tho a Field can only have one. So it should never
@@ -289,7 +300,8 @@
:effective-type (keyword v)
:fingerprint (if (map? v)
(perf/keywordize-keys v)
- (js->clj v :keywordize-keys true))
+ #?(:cljs (js->clj v :keywordize-keys true)
+ :clj v))
:has-field-values (keyword v)
;; Field refs are JS arrays, which we do not alter but do need to clone.
@@ -301,7 +313,7 @@
;; a key like `closure_uid_123456789` (the number is randomized at load time).
;; If the array has been frozen, that mutation will throw. So we clone the `:field-ref` array on its way into CLJS
;; land, and avoid the issue.
- :field-ref (to-array v)
+ :field-ref #?(:cljs (to-array v) :clj (legacy-mbql.normalize/normalize v))
:lib/source (case v
;; These are the legacy values from the "source" key
"aggregation" :source/aggregations
@@ -324,6 +336,8 @@
:lib/original-binning (parse-binning-info v)
::field-values (parse-field-values v)
::dimension (parse-dimension v)
+ :settings #?(:cljs (js->clj v :keywordize-keys true)
+ :clj (perf/keywordize-keys v))
v)))
(defmethod parse-object-fn* :field
@@ -373,30 +387,35 @@
(defmethod excluded-keys :card
[_object-type]
- #{:database
- :db
- :fks
- :metadata
- :metrics
- :plain-object
- :segments
- :schema
- :schema-name
- :table})
+ (-> #{:database
+ :db
+ :fks
+ :metadata
+ :metrics
+ :plain-object
+ :segments
+ :schema
+ :schema-name
+ :table}
+ #?(:cljs identity :clj (conj :fields))))
(defn- parse-fields [fields]
- (mapv (parse-object-fn :field) fields))
+ (perf/mapv (parse-object-fn :field) fields))
(defmethod parse-field-fn :card
[_object-type]
(fn [k v]
(case k
- :result-metadata (if ((some-fn sequential? array?) v)
+ :result-metadata (if (#?(:cljs (some-fn sequential? array?)
+ :clj sequential?) v)
(parse-fields v)
- (js->clj v :keywordize-keys true))
- :fields (parse-fields v)
+ #?(:cljs (js->clj v :keywordize-keys true)
+ :clj (perf/keywordize-keys v)))
+ :fields #?(:cljs (parse-fields v)
+ :clj (throw (ex-info ":fields" {})))
:visibility-type (keyword v)
- :dataset-query (js->clj v :keywordize-keys true)
+ :dataset-query #?(:cljs (js->clj v :keywordize-keys true)
+ :clj (perf/keywordize-keys v))
:type (keyword v)
;; this is not complete, add more stuff as needed.
v)))
@@ -436,9 +455,9 @@
[id (delay (assemble-card metadata id))]))
(-> #{}
(into (keep lib.util/legacy-string-table-id->card-id)
- (js-keys (object-get metadata "tables")))
+ (#?(:cljs js-keys :clj keys) (object-get metadata "tables")))
(into (map parse-long)
- (js-keys (object-get metadata "questions"))))))
+ (#?(:cljs js-keys :clj keys) (object-get metadata "questions"))))))
(defmethod lib-type :metric
[_object-type]
@@ -503,21 +522,23 @@
[_object-type]
(fn [k v]
(case k
- :template-tags (lib.normalize/normalize :metabase.lib.schema.template-tag/template-tag-map (js->clj v))
+ :template-tags (lib.normalize/normalize :metabase.lib.schema.template-tag/template-tag-map
+ #?(:cljs (js->clj v)
+ :clj v))
v)))
(defn- parse-objects-delay [object-type metadata]
(delay
(try
(parse-objects object-type metadata)
- (catch js/Error e
+ (catch #?(:cljs js/Error :clj Exception) e
(log/errorf e "Error parsing %s objects: %s" object-type (ex-message e))
nil))))
(defn- card->metric-card
[card]
(-> card
- (select-keys [:id :table-id :name :description :archived :dataset-query :source-card-id :type])
+ (perf/select-keys [:id :table-id :name :description :archived :dataset-query :source-card-id :type])
(assoc :lib/type :metadata/metric)))
(defn- metric-cards
@@ -569,7 +590,7 @@
(distinct))
delays)))
-(defn- setting [^js unparsed-metadata setting-key]
+(defn- setting [#?(:cljs ^js unparsed-metadata :clj unparsed-metadata) setting-key]
(-> unparsed-metadata
(object-get "settings")
(object-get (name setting-key))))
@@ -608,9 +629,10 @@
(defn metadata-provider
"Use a `metabase-lib/metadata/Metadata` as a [[metabase.lib.metadata.protocols/MetadataProvider]]."
[database-id unparsed-metadata]
- (lib.cache/side-channel-cache (str database-id) unparsed-metadata
- (partial metadata-provider* database-id)
- true #_force?))
+ #?(:cljs (lib.cache/side-channel-cache (str database-id) unparsed-metadata
+ (partial metadata-provider* database-id)
+ true #_force?)
+ :clj (metadata-provider* database-id unparsed-metadata)))
(def parse-column
"Parses a JS column provided by the FE into a :metadata/column value for use in MLv2."