The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
[perf] Tools to repro query + metadata locally, given instance API access (#67952)
Braden Shepherdsonb01732fd6904
diff --git a/dev/src/dev/debug_qp/analyze_user_query.clj b/dev/src/dev/debug_qp/analyze_user_query.cljnew file mode 100644index 00000000000..e322a7e5414--- /dev/null+++ b/dev/src/dev/debug_qp/analyze_user_query.clj@@ -0,0 +1,157 @@+(ns dev.debug-qp.analyze-user-query+ "Given access to e.g. a Cloud instance with a querying issue, this script can use a login token borrowed from your+ browser to make API calls to the instance, and assemble a local mock `MetadataProvider` which contains the tables,+ fields and cards necessary to run QP preprocessing and compilation locally against the same metadata the user+ instance is seeing.++ Of course the queries can't be executed without a connection to the user's DWH, which is usually impossible."+ (:require+ [clj-http.client :as http]+ [medley.core :as m]+ [metabase.lib.js.metadata :as lib.js.metadata]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.query-processor.compile :as qp.compile]+ [metabase.query-processor.preprocess :as qp.preprocess]+ [metabase.util.json :as json]+ [metabase.util.malli :as mu]))++;; XXX: INSTRUCTIONS:+;; - Log into the instance, open DevTools, and navigate to the problem question.+;; - Open the Network tab, find a `/api/card/1234` or similar request, right click and "Copy as cURL".+;; - Paste that somewhere, probably into your shell to test that the `curl` call is working.+;; - Grab its `cookie` and edit it into `my-ctx` below. (It's the `-b` arg to curl.)+;; - Edit the `instance-url` of `my-ctx` below likewise.+;; - Edit the `database-id` of `my-ctx` below to match the problem database on the user instance.+;; - That can be seen in eg. `/api/card` requests and similar.+;; - Now you can run the analysis functions below, like compile-card.+;; - See the examples at the bottom of this file.+(def ^:private my-ctx+ "Contains instance details, for passing to the various functions below."+ {:instance-url "https://acustomer.metabaseapp.com" ; No trailing slash.+ :cookie-from-browser "COPY ME FROM YOUR curl COMMAND"+ :database-id 123})++(defn- mk-headers [{:keys [cookie-from-browser instance-url] :as _ctx}]+ {"accept" "application/json"+ "accept-language" "en-US,en;q=0.9"+ "cache-control" "no-cache"+ "content-type" "application/json"+ "cookie" cookie-from-browser+ "pragma" "no-cache"+ "priority" "u=1, i"+ "referer" instance-url+ "sec-ch-ua" "\"Google Chrome\";v=\"141\", \"Not?A_Brand\";v=\"8\", \"Chromium\";v=\"141\""+ "sec-ch-ua-mobile" "?0"+ "sec-ch-ua-platform" "\"macOS\""+ "sec-fetch-dest" "empty"+ "sec-fetch-mode" "cors"+ "sec-fetch-site" "same-origin"+ "user-agent" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36"})++(defn- instance-fetch [{:keys [instance-url] :as ctx} path]+ (let [headers (mk-headers ctx)+ request {:method :get+ :url (str instance-url path)+ :accept :json+ :content-type :json+ :throw-exceptions false+ :headers headers}+ response (-> (http/request request)+ (select-keys [:body :headers :status])+ (update :body json/decode))]+ (if (<= 200 (:status response) 299)+ (:body response)+ (throw (ex-info "Non-200 response" response)))))++(defn- qm-index [objs]+ (m/index-by (comp str #(get % "id")) objs))++(defn- qm->metadata [raw]+ ;; /query_metadata requests return a map in a different shape from what lib.js.metadata expects.+ ;; We get {"databases" [{...}], "tables" [{..., "fields": [{...}]}]} but we need+ ;; {"databases" {"12" {...}}+ ;; "tables" {"333" {...}}+ ;; "fields" {"4343" {...}}+ ;; "questions" {"12" {...}}}+ (let [fields (for [table (get raw "tables")+ :when (number? (get table "id"))+ field (get table "fields")+ :when (number? (get field "id"))]+ field)]+ {"databases" (qm-index (get raw "databases"))+ "tables" (qm-index (get raw "tables"))+ "fields" (qm-index fields)}))++(defn- recursive-card-metadata+ "Helper for [[metadata-for]]. Prefer calling that function instead."+ [ctx main-card-id]+ (loop [metadata {}+ cards-seen #{main-card-id}+ [card-id & cards] [main-card-id]]+ (if-not card-id+ metadata+ (let [card (instance-fetch ctx (str "/api/card/" card-id))+ raw (instance-fetch ctx (str "/api/card/" card-id "/query_metadata"))+ nested-cards (->> (get raw "tables")+ (map #(get % "id"))+ (filter string?)+ (map (comp Long/parseLong #(subs % 6)))+ (remove cards-seen))]+ (recur (m/deep-merge metadata+ {"questions" {(str card-id) card}}+ (qm->metadata raw))+ (into cards-seen nested-cards)+ (concat cards nested-cards))))))++(defn- table-metadata+ "Helper for [[metadata-for]]. Prefer calling that function instead."+ [ctx table-id]+ (qm->metadata {"tables" [(-> (instance-fetch ctx (str "/api/table/" table-id "/query_metadata"))+ (dissoc "db"))]}))++;; =============================================== External API ===================================================+(defn metadata-for+ "Given a map like `{:card [123 456], :table [789]}` returns a `MetadataProvider` with metadata for all the tables,+ cards and fields transitively required by the specified roots.++ Generally you only need a single `{:card [123]}` for the problem card you're investigating.++ Occasionally a broken card might reference fields by ID which are not really part of their query - then they're+ missing from the `/query_metadata` on that card and will be missed by this function unless you explicitly add+ `{:table [789]}` to the map of roots.++ Returns a complete `MetadataProvider` ready to use in a QP call."+ [{:keys [database-id] :as ctx} kind->ids]+ (->> (for [[kind ids] kind->ids+ id ids]+ (case kind+ :card (recursive-card-metadata ctx id)+ :table (table-metadata ctx id)))+ (apply m/deep-merge)+ (lib.js.metadata/metadata-provider database-id)))++(defn preprocess-card+ "Given a `MetadataProvider` built by [[metadata-for]] and a card ID, tries to preprocess that card."+ [metadata-provider card-id]+ (mu/disable-enforcement+ (-> (lib.metadata/card metadata-provider card-id)+ :dataset-query+ qp.preprocess/preprocess)))++(defn compile-card+ "Given a `MetadataProvider` built by [[metadata-for]] and a card ID, tries to compile that card to e.g. SQL."+ [metadata-provider card-id]+ (mu/disable-enforcement+ (-> (lib.metadata/card metadata-provider card-id)+ :dataset-query+ qp.compile/compile)))++(comment+ ;; Example calls+ (let [card-id 456+ mp (metadata-for my-ctx {:card [card-id]})]+ (compile-card mp card-id))++ (let [card-id 456+ mp (metadata-for my-ctx {:card [card-id]})]+ (preprocess-card mp card-id)))diff --git a/src/metabase/lib/js/metadata.cljs b/src/metabase/lib/js/metadata.cljcsimilarity index 82%rename from src/metabase/lib/js/metadata.cljsrename to src/metabase/lib/js/metadata.cljcindex b44b4ee2b6b..bfc6cf64a4f 100644--- a/src/metabase/lib/js/metadata.cljs+++ b/src/metabase/lib/js/metadata.cljc@@ -1,12 +1,14 @@(ns metabase.lib.js.metadata(:refer-clojure :exclude [keywordize-keys])(:require+ #?@(:clj (#_{:clj-kondo/ignore [:discouraged-namespace]}+ [metabase.legacy-mbql.normalize :as legacy-mbql.normalize])+ :cljs ([goog]+ [goog.object :as gobject]+ [metabase.lib.cache :as lib.cache]))[clojure.core.protocols][clojure.string :as str]- [goog]- [goog.object :as gobject][medley.core :as m]- [metabase.lib.cache :as lib.cache][metabase.lib.metadata.cached-provider :as lib.metadata.cached-provider][metabase.lib.metadata.protocols :as lib.metadata.protocols][metabase.lib.normalize :as lib.normalize]@@ -29,8 +31,10 @@;;; where keys are a map of String ID => metadata(defn- object-get [obj k]- (when (and obj (js-in k obj))- (gobject/get obj k)))+ #?(:cljs (when (and obj (js-in k obj))+ (gobject/get obj k))+ :clj (when (and obj (contains? obj k))+ (get obj k))))(defn- obj->clj"Convert a JS object of *any* class to a ClojureScript object."@@ -43,23 +47,26 @@;; already a ClojureScript object.(into {} xform (m/remove-keys skip-keys obj));; has a plain-JavaScript `_plainObject` attached: apply `xform` to it and call it a day- (if-let [plain-object (when use-plain-object?- (some-> (object-get obj "_plainObject")- js->clj- not-empty))]- (into {} xform (m/remove-keys skip-keys plain-object))- ;; otherwise do things the hard way and convert an arbitrary object into a Cljs map. (`js->clj` doesn't work on- ;; arbitrary classes other than `Object`)- (into {}- (comp- (remove (set skip-keys))- (map (fn [k]- [k (object-get obj k)]))- ;; ignore values that are functions- (remove (fn [[_k v]]- (js-fn? v)))- xform)- (js-keys obj))))))+ #?(:cljs (if-let [plain-object (when use-plain-object?+ (some-> (object-get obj "_plainObject")+ js->clj+ not-empty))]+ (into {} xform (m/remove-keys skip-keys plain-object))+ ;; otherwise do things the hard way and convert an arbitrary object into a Cljs map. (`js->clj` doesn't work on+ ;; arbitrary classes other than `Object`)+ (into {}+ (comp+ (remove (set skip-keys))+ (map (fn [k]+ [k (object-get obj k)]))+ ;; ignore values that are functions+ (remove (fn [[_k v]]+ (js-fn? v)))+ xform)+ (js-keys obj)))+ :clj (when obj+ (comment use-plain-object?) ; Touch this binding to make linting happy.+ (throw (ex-info "_plainObject classy nesting not supported in CLJ" {:obj obj})))))));;; this intentionally does not use the lib hierarchy since it's not dealing with MBQL/lib keys(defmulti ^:private excluded-keys@@ -96,6 +103,7 @@Use [[excluded-keys]] to drop keys from the input.Defaults to nil, which means no renaming is done."+ {:arglists '([object-type])}identity)(defmethod rename-key-fn :default [_]@@ -112,7 +120,7 @@rename-key (#(or (rename-key %) %)))v]));; remove [[excluded-keys]]- (if (empty? excluded-keys-set)+ (if (perf/empty? excluded-keys-set)identity(remove (fn [[k _v]](contains? excluded-keys-set k))))@@ -141,7 +149,7 @@(let [parsed (assoc (obj->clj xform object opts) :lib/type lib-type-name)](log/debugf "Parsed metadata %s %s\n%s" object-type (:id parsed) (u/pprint-to-str parsed))parsed)- (catch js/Error e+ (catch #?(:cljs js/Error :clj Exception) e(log/errorf e "Error parsing %s %s: %s" object-type (pr-str object) (ex-message e))nil)))))@@ -175,10 +183,13 @@[_object-type](fn [k v](case k- :dbms-version (js->clj v :keywordize-keys true)+ :dbms-version #?(:cljs (js->clj v :keywordize-keys true)+ :clj (perf/keywordize-keys v))+ :engine (keyword v):features (into #{} (map keyword) v):native-permissions (keyword v)- :settings (js->clj v :keywordize-keys true)+ :settings #?(:cljs (js->clj v :keywordize-keys true)+ :clj (perf/keywordize-keys v))v)))(defmethod parse-objects-default-key :database@@ -255,8 +266,8 @@(defn- parse-field-values [field-values](when (= (object-get field-values "type") "full")- {:values (js->clj (object-get field-values "values"))- :human-readable-values (js->clj (object-get field-values "human_readable_values"))}))+ {:values (#?(:cljs js->clj :clj identity) (object-get field-values "values"))+ :human-readable-values (#?(:cljs js->clj :clj identity) (object-get field-values "human_readable_values"))}))(defn- parse-dimension"`:dimensions` comes in as an array for historical reasons, even tho a Field can only have one. So it should never@@ -289,7 +300,8 @@:effective-type (keyword v):fingerprint (if (map? v)(perf/keywordize-keys v)- (js->clj v :keywordize-keys true))+ #?(:cljs (js->clj v :keywordize-keys true)+ :clj v)):has-field-values (keyword v);; Field refs are JS arrays, which we do not alter but do need to clone.@@ -301,7 +313,7 @@;; a key like `closure_uid_123456789` (the number is randomized at load time).;; If the array has been frozen, that mutation will throw. So we clone the `:field-ref` array on its way into CLJS;; land, and avoid the issue.- :field-ref (to-array v)+ :field-ref #?(:cljs (to-array v) :clj (legacy-mbql.normalize/normalize v)):lib/source (case v;; These are the legacy values from the "source" key"aggregation" :source/aggregations@@ -324,6 +336,8 @@:lib/original-binning (parse-binning-info v)::field-values (parse-field-values v)::dimension (parse-dimension v)+ :settings #?(:cljs (js->clj v :keywordize-keys true)+ :clj (perf/keywordize-keys v))v)))(defmethod parse-object-fn* :field@@ -373,30 +387,35 @@(defmethod excluded-keys :card[_object-type]- #{:database- :db- :fks- :metadata- :metrics- :plain-object- :segments- :schema- :schema-name- :table})+ (-> #{:database+ :db+ :fks+ :metadata+ :metrics+ :plain-object+ :segments+ :schema+ :schema-name+ :table}+ #?(:cljs identity :clj (conj :fields))))(defn- parse-fields [fields]- (mapv (parse-object-fn :field) fields))+ (perf/mapv (parse-object-fn :field) fields))(defmethod parse-field-fn :card[_object-type](fn [k v](case k- :result-metadata (if ((some-fn sequential? array?) v)+ :result-metadata (if (#?(:cljs (some-fn sequential? array?)+ :clj sequential?) v)(parse-fields v)- (js->clj v :keywordize-keys true))- :fields (parse-fields v)+ #?(:cljs (js->clj v :keywordize-keys true)+ :clj (perf/keywordize-keys v)))+ :fields #?(:cljs (parse-fields v)+ :clj (throw (ex-info ":fields" {}))):visibility-type (keyword v)- :dataset-query (js->clj v :keywordize-keys true)+ :dataset-query #?(:cljs (js->clj v :keywordize-keys true)+ :clj (perf/keywordize-keys v)):type (keyword v);; this is not complete, add more stuff as needed.v)))@@ -436,9 +455,9 @@[id (delay (assemble-card metadata id))]))(-> #{}(into (keep lib.util/legacy-string-table-id->card-id)- (js-keys (object-get metadata "tables")))+ (#?(:cljs js-keys :clj keys) (object-get metadata "tables")))(into (map parse-long)- (js-keys (object-get metadata "questions"))))))+ (#?(:cljs js-keys :clj keys) (object-get metadata "questions"))))))(defmethod lib-type :metric[_object-type]@@ -503,21 +522,23 @@[_object-type](fn [k v](case k- :template-tags (lib.normalize/normalize :metabase.lib.schema.template-tag/template-tag-map (js->clj v))+ :template-tags (lib.normalize/normalize :metabase.lib.schema.template-tag/template-tag-map+ #?(:cljs (js->clj v)+ :clj v))v)))(defn- parse-objects-delay [object-type metadata](delay(try(parse-objects object-type metadata)- (catch js/Error e+ (catch #?(:cljs js/Error :clj Exception) e(log/errorf e "Error parsing %s objects: %s" object-type (ex-message e))nil))))(defn- card->metric-card[card](-> card- (select-keys [:id :table-id :name :description :archived :dataset-query :source-card-id :type])+ (perf/select-keys [:id :table-id :name :description :archived :dataset-query :source-card-id :type])(assoc :lib/type :metadata/metric)))(defn- metric-cards@@ -569,7 +590,7 @@(distinct))delays)))-(defn- setting [^js unparsed-metadata setting-key]+(defn- setting [#?(:cljs ^js unparsed-metadata :clj unparsed-metadata) setting-key](-> unparsed-metadata(object-get "settings")(object-get (name setting-key))))@@ -608,9 +629,10 @@(defn metadata-provider"Use a `metabase-lib/metadata/Metadata` as a [[metabase.lib.metadata.protocols/MetadataProvider]]."[database-id unparsed-metadata]- (lib.cache/side-channel-cache (str database-id) unparsed-metadata- (partial metadata-provider* database-id)- true #_force?))+ #?(:cljs (lib.cache/side-channel-cache (str database-id) unparsed-metadata+ (partial metadata-provider* database-id)+ true #_force?)+ :clj (metadata-provider* database-id unparsed-metadata)))(def parse-column"Parses a JS column provided by the FE into a :metadata/column value for use in MLv2."