The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Use Lib for `query->source-card`; fix busted tests and busted X-Ray code (#63921)
Cam Sauld018b4a1bc31
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 16db4768ab3..5d92ee979bf 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -939,6 +939,7 @@:uses #{apilegacy-mbqllib+ lib-bemodelspermissionsquery-processor@@ -1472,6 +1473,7 @@driverlegacy-mbqllib+ lib-bemodelsqueriesquery-permissionsdiff --git a/enterprise/backend/test/metabase_enterprise/cache/task/refresh_cache_configs_test.clj b/enterprise/backend/test/metabase_enterprise/cache/task/refresh_cache_configs_test.cljindex fb68b004d07..f9804684564 100644--- a/enterprise/backend/test/metabase_enterprise/cache/task/refresh_cache_configs_test.clj+++ b/enterprise/backend/test/metabase_enterprise/cache/task/refresh_cache_configs_test.clj@@ -84,9 +84,9 @@(deftest schedule-cache-config->card-ids-test(mt/with-temp [:model/Dashboard {dashboard-id :id} {:name "Dashboard"}:model/Card {card-id-1 :id} {:name "Cached card 1"- :dataset_query {:database 1}}+ :dataset_query {:database (mt/id), :type :native, :native {:query "SELECT 1;"}}}:model/Card {card-id-2 :id} {:name "Cached card 2"- :dataset_query {:database 1}}+ :dataset_query {:database (mt/id), :type :native, :native {:query "SELECT 1;"}}}:model/DashboardCard {} {:dashboard_id dashboard-id:card_id card-id-1}:model/DashboardCard {} {:dashboard_id dashboard-id@@ -97,7 +97,6 @@:card_id card-id-2}](testing "Returns the card ID directly for a question cache config"(is (= [card-id-1] (@#'task.cache/schedule-cache-config->card-ids {:model_id card-id-1 :model "question"}))))-(testing "Fetches card IDs associated with a dashboard cache config"(is (= [card-id-1 card-id-2] (@#'task.cache/schedule-cache-config->card-ids {:model_id dashboard-id :model "dashboard"}))))))@@ -137,7 +136,7 @@(is (= [nil param-val-2] (map param-vals (to-rerun))))))))))))(deftest scheduled-base-query-to-rerun-edge-cases-test- (let [query {:database 1}]+ (let [query {:database (mt/id), :type :native, :native {:query "SELECT 1;"}}](mt/with-temp [:model/Card {card-id :id} {:name "Cached card":dataset_query query}:model/Query {} {:query_hash (qp.util/query-hash query)@@ -170,7 +169,7 @@(is (= [] (t2/select :model/Query (@#'task.cache/scheduled-base-query-to-rerun-honeysql card-id)))))))))(deftest scheduled-parameterized-queries-to-rerun-edge-cases-test- (let [query {:database 1}]+ (let [query {:database (mt/id), :type :native, :native {:query "SELECT 1;"}}](mt/with-temp [:model/Card {card-id :id} {:name "Cached card":dataset_query query}:model/Query {} {:query_hash (qp.util/query-hash query)@@ -299,8 +298,8 @@(is (= [nil param-val-2] (map param-vals (to-rerun card-id))))))))))))(deftest duration-base-queries-to-rerun-edge-cases-test- (let [query-1 {:database 1}- query-2 {:database 2}+ (let [query-1 {:database (mt/id), :type "native", :native {:query "SELECT * FROM x"}}+ query-2 {:database (mt/id), :type "native", :native {:query "SELECT * FROM y"}}query-1-cache-hash (qp.util/query-hash query-1)query-2-cache-hash (qp.util/query-hash query-2)](mt/with-temp [:model/Dashboard {dashboard-id :id} {:name "Dashboard"}@@ -390,8 +389,8 @@[question-cache-config-1] false))))))))))(deftest duration-parameterized-queries-to-rerun-edge-cases-test- (let [query-1 {:database 1}- query-2 {:database 2}+ (let [query-1 {:database (mt/id), :type "native", :native {:query "SELECT * FROM x"}}+ query-2 {:database (mt/id), :type "native", :native {:query "SELECT * FROM y"}}query-1-cache-hash (qp.util/query-hash query-1)query-2-cache-hash (qp.util/query-hash query-2)](mt/with-temp [:model/Dashboard {dashboard-id :id} {:name "Dashboard"}diff --git a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/api_test.cljindex c2e5be24b62..ba4f0529148 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj@@ -26,7 +26,7 @@(GzipCompressorInputStream.)(TarArchiveInputStream.)))-(def FILE-TYPES+(def ^:private file-types[#"([^/]+)?/$" :dir#"/settings.yaml$" :settings#"/export.log$" :log@@ -46,7 +46,7 @@(some (fn [[re ftype]](when-let [m (re-find re fname)][ftype (when (vector? m) (second m))]))- (partition 2 FILE-TYPES)))+ (partition 2 file-types)))(defn- log-types"Find out entity type by log message"@@ -74,7 +74,7 @@(assoc m k "**ID**")m)))-(defn extract-and-sanitize-exception-map [log]+(defn- extract-and-sanitize-exception-map [log](->> (re-find #"ERROR .* (\{.*\})(\n|$)" log)secondread-stringdiff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.cljindex 751b7828630..a156418dac7 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj@@ -163,6 +163,7 @@:dataset_query {:query {:source-table no-schema-id:filter [:>= [:field field-id nil] 18]:aggregation [[:count]]}+ :type :query:database db-id}}:model/Card@@ -176,6 +177,7 @@:dataset_query {:query {:source-table no-schema-id:filter [:>= [:field field-id nil] 18]:aggregation [[:count]]}+ :type :query:database db-id}}:model/Card@@ -225,24 +227,26 @@:model/Card {c4-id :idc4-eid :entity_id} {:name "Referenced Question":database_id db-id- :table_id schema-id+ :table_id no-schema-id:collection_id coll-id:creator_id mark-id:dataset_query{:query {:source-table no-schema-id:filter [:>= [:field field-id nil] 18]}- :database db-id}}+ :database db-id+ :type :query}}:model/Card{c5-id :idc5-eid :entity_id}{:name "Dependent Question":database_id db-id- :table_id schema-id+ :table_id no-schema-id:collection_id coll-id:creator_id mark-id:dataset_query{:query {:source-table (str "card__" c4-id):aggregation [[:count]]}+ :type :query:database db-id}}:model/Action@@ -424,7 +428,7 @@(testing "Cards can be based on other cards"(let [ser (serdes/extract-one "Card" {} (t2/select-one :model/Card :id c5-id))](is (=? {:serdes/meta [{:model "Card" :id c5-eid :label "dependent_question"}]- :table_id ["My Database" "PUBLIC" "Schema'd Table"]+ :table_id ["My Database" nil "Schemaless Table"]:creator_id "[email protected]":collection_id coll-eid:dataset_query {:query {:source-table c4-eid@@ -437,8 +441,7 @@(testing "and depend on their Database, Table and Collection, and the upstream Card"(is (= #{[{:model "Database" :id "My Database"}][{:model "Database" :id "My Database"}- {:model "Schema" :id "PUBLIC"}- {:model "Table" :id "Schema'd Table"}]+ {:model "Table" :id "Schemaless Table"}][{:model "Collection" :id coll-eid}][{:model "Card" :id c4-eid}]}(set (serdes/dependencies ser)))))))@@ -1516,18 +1519,23 @@:model/Collection {coll2-id :id} {:name "Other Collection"}:model/Collection {coll3-id :id} {:name "Third Collection"}:model/Dashboard {dash-id :id} {:name "A Dashboard" :collection_id coll1-id}- :model/Card {card1-id :id} {:name "Some Card"}+ :model/Database {db-id :id} {}+ :model/Card {card1-id :id} {:name "Some Card", :database_id db-id}:model/DashboardCard _ {:card_id card1-id :dashboard_id dash-id}:model/Card _ {:name "Dependent Card":collection_id coll2-id- :dataset_query {:query {:source-table (str "card__" card1-id)}}}+ :dataset_query {:type :query+ :database db-id+ :query {:source-table (str "card__" card1-id)}}}:model/User user {:email "[email protected]"}:model/Collection pcoll {:name "Personal Collection":personal_owner_id (:id user)}:model/Card pcard {:name "Personal Card":collection_id (:id pcoll)}:model/Card _ {:name "External Card"- :dataset_query {:query {:source-table (str "card__" (:id pcard))}}}+ :dataset_query {:database db-id+ :type :query+ :query {:source-table (str "card__" (:id pcard))}}}:model/Card _ {:name "Card with parameters":collection_id coll3-id:parameters [{:id "abc"diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.cljindex 7ceb7f790b7..24154306cfd 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj@@ -903,13 +903,15 @@(reset! card1s (ts/create! :model/Card:name "the query":dataset_query {:database (:id @db1s)+ :type :native:native {:template-tags {"snippet: things"- {:id "e2d15f07-37b3-01fc-3944-2ff860a5eb46",- :name "snippet: filtered data",- :display-name "Snippet: Filtered Data",- :type :snippet,- :snippet-name "filtered data",- :snippet-id (:id @snippet1s)}}}}))+ {:id "e2d15f07-37b3-01fc-3944-2ff860a5eb46"+ :name "snippet: filtered data"+ :display-name "Snippet: Filtered Data"+ :type :snippet+ :snippet-name "filtered data"+ :snippet-id (:id @snippet1s)}}+ :query "SELECT 1;"}}))(ts/create! :model/User :first_name "Geddy" :last_name "Lee" :email "[email protected]")(testing "on extraction"@@ -1186,8 +1188,8 @@:type :model:database_id (:id db):dataset_query {:database (:id db)- :native {:type :native- :native {:query "wow"}}})+ :type :native+ :native {:query "wow"}})parent (ts/create! :model/Collection :name "Parent Collection" :location "/")_child (ts/create! :model/Collection:name "Child Collection"@@ -1449,7 +1451,7 @@{:model "Field" :id "field"}]}(ts/extract-one "Field" (:id f2))))- (is (=? {:parent_id ["mydb" nil "table" "field" "field"],+ (is (=? {:parent_id ["mydb" nil "table" "field" "field"]:serdes/meta [{:model "Database" :id "mydb"}{:model "Table" :id "table"}{:model "Field" :id "field"}diff --git a/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj b/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.cljindex 68f9c7be5fb..d9f4bb603e3 100644--- a/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj+++ b/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj@@ -1779,7 +1779,7 @@ function(bin) {(let [query (update query :query preprocess)](binding [*query* query*next-alias-index* (volatile! 0)]- (let [source-table-name (if-let [source-table-id (driver-api/query->source-table-id query)]+ (let [source-table-name (if-let [source-table-id #_{:clj-kondo/ignore [:deprecated-var]} (driver-api/query->source-table-id query)](:name (driver-api/table (driver-api/metadata-provider) source-table-id))(query->collection-name query))compiled (mbql->native-rec (:query query))]diff --git a/src/metabase/api/open_api.clj b/src/metabase/api/open_api.cljindex 79fe54da38e..61970c6c0b2 100644--- a/src/metabase/api/open_api.clj+++ b/src/metabase/api/open_api.clj@@ -98,7 +98,7 @@[:map[:type [:= :string]]- [:format {:optional true} [:enum :binary "binary" :byte "byte"]]+ [:format {:optional true} [:enum :binary "binary" :byte "byte" :uuid "uuid"]][:minLength {:optional true} integer?][:maxLength {:optional true} integer?][:pattern {:optional true} (ms/InstanceOfClass java.util.regex.Pattern)]]])diff --git a/src/metabase/legacy_mbql/schema.cljc b/src/metabase/legacy_mbql/schema.cljcindex b7e82663b85..cd57da6134d 100644--- a/src/metabase/legacy_mbql/schema.cljc+++ b/src/metabase/legacy_mbql/schema.cljc@@ -1209,7 +1209,7 @@[:display-name ::lib.schema.common/non-blank-string];; TODO -- `:id` is actually 100% required but we have a lot of tests that don't specify it because this constraint;; wasn't previously enforced; we need to go in and fix those tests and make this non-optional- [:id {:optional true} ::lib.schema.common/non-blank-string]])+ [:id {:optional true} [:ref ::lib.schema.template-tag/id]]]);; Example:;;diff --git a/src/metabase/legacy_mbql/util.cljc b/src/metabase/legacy_mbql/util.cljcindex 1442cf5d917..8e6c04f80d5 100644--- a/src/metabase/legacy_mbql/util.cljc+++ b/src/metabase/legacy_mbql/util.cljc@@ -523,8 +523,10 @@`nil`, returns `nil`.Throws an Exception when it encounters a unresolved source query (i.e., the `:source-table \"card__id\"`- form), because it cannot return an accurate result for a query that has not yet been preprocessed."- {:arglists '([outer-query])}+ form), because it cannot return an accurate result for a query that has not yet been preprocessed.++ Prefer [[metabase.lib.core/source-table-id]] going forward."+ {:arglists '([outer-query]), :deprecated "0.57.0"}[{{source-table-id :source-table, source-query :source-query} :query, query-type :type, :as query} :- [:maybe :map]](cond;; for native queries, there's no source table to resolve@@ -717,6 +719,7 @@(uniquify-names [\"count\" \"sum\" \"count\" \"count_2\"]);; -> [\"count\" \"sum\" \"count_2\" \"count_2_2\"]"+ {:deprecated "0.57.0"}[names :- [:sequential :string]](map (unique-name-generator) names))@@ -803,19 +806,13 @@(log/warnf "%s is not a valid temporal unit for %s; not adding to clause %s" unit base-type (pr-str clause))clause)))-(defn remove-namespaced-options- "Update a `:field`, `:expression` reference, or `:aggregation` reference clause by removing all namespaced keys in the- options map. This is mainly for clause equality comparison purposes -- in current usage namespaced keys are used by- individual pieces of middleware or driver implementations for tracking little bits of information that should not be- considered relevant when comparing clauses for equality."- [field-or-ref]- (update-field-options field-or-ref (partial into {} (remove (fn [[k _]]- (qualified-keyword? k))))))-(defn referenced-field-ids"Find all the `:field` references with integer IDs in `coll`, which can be a full MBQL query, a snippet of MBQL, or asequence of those things; return a set of Field IDs. Includes Fields referenced indirectly via `:source-field`.- Returns `nil` if no IDs are found."+ Returns `nil` if no IDs are found.++ DEPRECATED: Use [[metabase.lib.core/all-field-ids]] going forward."+ {:deprecated "0.57.0"}[coll](not-empty(into #{}diff --git a/src/metabase/lib/metadata/result_metadata.cljc b/src/metabase/lib/metadata/result_metadata.cljcindex ff18aa5c34a..fc438b2c4c2 100644--- a/src/metabase/lib/metadata/result_metadata.cljc+++ b/src/metabase/lib/metadata/result_metadata.cljc@@ -230,6 +230,15 @@(assoc col :source (legacy-source col)))cols))+(defn- remove-namespaced-options+ "Update a legacy `:field`, `:expression` reference, or `:aggregation` reference clause by removing all namespaced keys+ in the options map. This is mainly for clause equality comparison purposes -- in current usage namespaced keys are+ used by individual pieces of middleware or driver implementations for tracking little bits of information that+ should not be considered relevant when comparing clauses for equality."+ [legacy-ref]+ (mbql.u/update-field-options legacy-ref (partial into {} (remove (fn [[k _]]+ (qualified-keyword? k))))))+(mu/defn- fe-friendly-expression-ref :- ::mbql.s/Reference"Apparently the FE viz code breaks for pivot queries if `field_ref` comes back with extra 'non-traditional' Libinfo (`:base-type` or `:effective-type` in `:expression`), so we better just strip this info out to be sure. If you@@ -237,7 +246,7 @@will see."[col :- ::kebab-cased-mapa-ref :- ::mbql.s/Reference]- (let [a-ref (mbql.u/remove-namespaced-options a-ref)]+ (let [a-ref (remove-namespaced-options a-ref)](lib.util.match/replace a-ref[:field (id :guard pos-int?) opts][:field id (not-empty (cond-> (dissoc opts :effective-type :inherited-temporal-unit)diff --git a/src/metabase/lib/schema.cljc b/src/metabase/lib/schema.cljcindex 3600914ebe7..d05759fbde6 100644--- a/src/metabase/lib/schema.cljc+++ b/src/metabase/lib/schema.cljc@@ -305,7 +305,8 @@See [[metabase.driver.sql.query-processor-test/join-source-queries-with-joins-test]] for example.This doesn't really make sense IMO (you should use string field refs to refer to things from a previous- stage...right?) but for now we'll have to allow it until we can figure out how to go fix all of the old broken queries.+ stage...right?) but for now we'll have to allow it until we can figure out how to go fix all of the old broken+ queries.Also, it's apparently legal to use a join alias to refer to a column that comes from a join in a source Card, andthere is no way for us to know what joins exist in the source Card without a metadata provider, so we're just going@@ -326,8 +327,18 @@(mapcat join-aliases-in-join (:joins stage)))](set (join-aliases-in-stage stage)))))-(defn- join-ref-error-for-stages [stages]- (when (sequential? stages)+(def ^:dynamic *HACK-disable-join-alias-in-field-ref-validation*+ "Whether to validate join aliases in field refs. This is only disable-able as a hack to support X-Rays code which+ generates fragments of stages that drop joins and then adds them again after the fact+ in [[metabase.xrays.automagic-dashboards.core/preserve-entity-element]]. Once we port X-Rays to use Lib we can fix+ the hackiness and hopefully take this out."+ false)++(defn- join-ref-error-for-stages+ "Return an error messages if we find a field ref that uses a `:join-alias` for a join that doesn't exist."+ [stages]+ (when (and (not *HACK-disable-join-alias-in-field-ref-validation*)+ (sequential? stages))(loop [visible-join-alias? (constantly false), i 0, [stage & more] stages](let [visible-join-alias? (some-fn visible-join-alias? (visible-join-alias?-fn stage))](or@@ -339,20 +350,12 @@(when (seq more)(recur visible-join-alias? (inc i) more)))))))-(def ^:private ^{:arglists '([stages])} ref-error-for-stages- "Like [[ref-error-for-stage]], but validate references in the context of a sequence of several stages; for validations- that can't be done on the basis of just a single stage. For example join alias validation needs to take into account- previous stages."- ;; this var is ultimately redundant for now since it just points to one function but I'm leaving it here so we can- ;; add more stuff to it the future as we validate more things.- join-ref-error-for-stages)-(mr/def ::stages.valid-refs[:fn{:error/message "Valid references for all query stages":error/fn (fn [{stages :value} _]- (ref-error-for-stages stages))}- (complement ref-error-for-stages)])+ (join-ref-error-for-stages stages))}+ (complement #'join-ref-error-for-stages)])(defn- normalize-stages [stages](when (sequential? stages)@@ -379,6 +382,11 @@[:* [:schema [:ref ::stage.additional]]]][:ref ::stages.valid-refs]])+(defn- normalize-query [query]+ (when-let [query (common/normalize-map query)]+ (cond-> query+ (not (:lib/metadata query)) (dissoc :lib/metadata))))+(defn- serialize-query [query];; this stuff all gets added in when you actually run a query with one of the QP entrypoints, and is not considered;; to be part of the query itself. It doesn't get saved along with the query in the app DB.@@ -395,8 +403,8 @@[:and[:map{:description "Valid MBQL 5 query."- :decode/normalize common/normalize-map- :encode/serialize serialize-query}+ :decode/normalize #'normalize-query+ :encode/serialize #'serialize-query}[:lib/type [:={:decode/normalize common/normalize-keyword, :default :mbql/query}:mbql/query]]diff --git a/src/metabase/lib_be/metadata/bootstrap.clj b/src/metabase/lib_be/metadata/bootstrap.cljindex 75007d46d54..b8d8f811e92 100644--- a/src/metabase/lib_be/metadata/bootstrap.clj+++ b/src/metabase/lib_be/metadata/bootstrap.clj@@ -64,8 +64,9 @@(->BootstrapMetadataProvider))(mu/defn- resolve-database-id-for-source-card :- ::lib.schema.id/database- [source-card-id :- ::lib.schema.id/card]- (let [card (or (lib.metadata.protocols/card (bootstrap-metadata-provider) source-card-id)+ [metadata-provider :- [:maybe ::lib.metadata.protocols/metadata-provider]+ source-card-id :- ::lib.schema.id/card]+ (let [card (or (lib.metadata.protocols/card (or metadata-provider (bootstrap-metadata-provider)) source-card-id)(throw (ex-info (i18n/tru "Card {0} does not exist." source-card-id){:card-id source-card-id, :type :invalid-query, :status-code 404})))](:database-id card)))@@ -84,7 +85,8 @@{:query query, :type :invalid-query}))))(mu/defn- resolved-database-id :- [:maybe ::lib.schema.id/database]- [query :- :map]+ [metadata-provider :- [:maybe ::lib.metadata.protocols/metadata-provider]+ query :- :map](when-not (= (query-type query) :internal)(let [database-id (:database query)](cond@@ -92,7 +94,7 @@database-id(= database-id lib.schema.id/saved-questions-virtual-database-id)- (resolve-database-id-for-source-card (source-card-id query))+ (resolve-database-id-for-source-card metadata-provider (source-card-id query)):else(throw (ex-info (i18n/tru "Invalid query: missing or invalid Database ID (:database)")@@ -106,9 +108,21 @@[:type [:= :internal]]]]]"If query has `:database` `-1337` (the legacy database ID for queries using a source Card that had an unknowndatabase), resolve the correct database ID and assoc it into the query."- [query :- [:maybe :map]]- (when (seq query)- (let [query (set/rename-keys query {"database" :database})- database-id (resolved-database-id query)]- (cond-> query- database-id (assoc :database database-id)))))+ ([query]+ (resolve-database nil query))++ ([metadata-provider :- [:maybe ::lib.metadata.protocols/metadata-provider]+ query :- [:maybe+ [:and+ :map+ [:multi {:dispatch (comp boolean empty?)}+ [true [:= {:description "empty map"} {}]]+ [false [:map+ [:database [:or+ ::lib.schema.id/database+ ::lib.schema.id/saved-questions-virtual-database]]]]]]]]+ (when (seq query)+ (let [query (set/rename-keys query {"database" :database})+ database-id (resolved-database-id metadata-provider query)]+ (cond-> query+ database-id (assoc :database database-id))))))diff --git a/src/metabase/lib_be/models/transforms.clj b/src/metabase/lib_be/models/transforms.cljindex 0b29813d0ef..fbf5935930b 100644--- a/src/metabase/lib_be/models/transforms.clj+++ b/src/metabase/lib_be/models/transforms.clj@@ -3,23 +3,34 @@[metabase.lib-be.metadata.bootstrap :as lib-be.bootstrap][metabase.lib-be.metadata.jvm :as lib.metadata.jvm][metabase.lib.core :as lib]+ [metabase.lib.metadata.protocols :as lib.metadata.protocols][metabase.models.interface :as mi][metabase.util.malli :as mu]))(mu/defn normalize-query"Normalize an MBQL `query` to MBQL 5 and attach a metadata provider."- [query :- [:maybe :map]]- (cond- (empty? query)- {}+ ([query]+ (normalize-query nil query))- (lib/cached-metadata-provider-with-cache? (:lib/metadata query))- query+ ([metadata-providerable :- [:maybe ::lib.metadata.protocols/metadata-providerable]+ query :- [:maybe :map]]+ (let [metadata-providerable (or metadata-providerable+ (when-let [mp (:lib/metadata query)]+ (when (lib/metadata-provider? mp)+ mp))+ lib.metadata.jvm/application-database-metadata-provider)]+ (cond+ (empty? query)+ {}- :else- (->> query- lib-be.bootstrap/resolve-database- (lib/query lib.metadata.jvm/application-database-metadata-provider))))+ (lib/cached-metadata-provider-with-cache? (:lib/metadata query))+ query++ :else+ (->> query+ (lib-be.bootstrap/resolve-database (when (lib/metadata-provider? metadata-providerable)+ metadata-providerable))+ (lib/query metadata-providerable))))))(def transform-query"Toucan 2 transform spec for Card `dataset_query` and other columns that store MBQL."diff --git a/src/metabase/parameters/params.clj b/src/metabase/parameters/params.cljindex 87bde826503..47465266d2f 100644--- a/src/metabase/parameters/params.clj+++ b/src/metabase/parameters/params.clj@@ -357,7 +357,7 @@[card](into {} (for [[_ {param-id :iddimension :dimension}] (get-in card [:dataset_query :native :template-tags])- :when dimension+ :when (and dimension param-id):let [field (mbql.u/unwrap-field-clause dimension)]:when field][param-id #{field}])))diff --git a/src/metabase/queries/api/card.clj b/src/metabase/queries/api/card.cljindex 4f420a0e580..7526b0e2978 100644--- a/src/metabase/queries/api/card.clj+++ b/src/metabase/queries/api/card.clj@@ -10,7 +10,7 @@[metabase.eid-translation.core :as eid-translation][metabase.embedding.validation :as embedding.validation][metabase.events.core :as events]- [metabase.lib-be.metadata.jvm :as lib.metadata.jvm]+ [metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.types.isa :as lib.types.isa][metabase.models.interface :as mi]@@ -99,21 +99,22 @@:where [:and [:= :m.id model-id] [:not :c.archived]]:order-by [[[:lower :c.name] :asc]]});; now check if model-id really occurs as a card ID+ ;;+ ;; existing usage -- do not use in new code+ #_{:clj-kondo/ignore [:deprecated-var]}(filter (fn [card] (some #{model-id} (-> card :dataset_query query/collect-card-ids))))))(defn- cards-for-segment-or-metric[model-type model-id]- (lib.metadata.jvm/with-metadata-provider-cache+ (lib-be/with-metadata-provider-cache(->> (t2/select :model/Card (merge order-by-name{:where [:like :dataset_query (str "%" (name model-type) "%" model-id "%")]}));; now check if the segment/metric with model-id really occurs in a filter/aggregation expression(filter (fn [card]- (when-let [legacy-query (some-> card :dataset_query)]- (let [mp (lib.metadata.jvm/application-database-metadata-provider (:database_id card))- query (lib/query mp legacy-query)]- (case model-type- :segment (lib/uses-segment? query model-id)- :metric (lib/uses-metric? query model-id)))))))))+ (when-let [query (some-> card :dataset_query not-empty lib-be/normalize-query)]+ (case model-type+ :segment (lib/uses-segment? query model-id)+ :metric (lib/uses-metric? query model-id))))))))(defmethod cards-for-filter-option* :using_segment[_filter-option model-id]@@ -256,7 +257,7 @@"Convert the `dataset_query` column of a Card to a MLv2 pMBQL query."([dataset-query](some-> (:database dataset-query)- lib.metadata.jvm/application-database-metadata-provider+ lib-be/application-database-metadata-provider(dataset-query->query dataset-query)))([metadata-provider dataset-query](some->> dataset-query card.metadata/normalize-dataset-query (lib/query metadata-provider))))@@ -377,7 +378,7 @@(set)(remove #(contains? database-ids %))(into database-id->metadata-provider- (map (juxt identity lib.metadata.jvm/application-database-metadata-provider))))+ (map (juxt identity lib-be/application-database-metadata-provider))))compatible-cards (->> matching-cards(filter mi/can-read?)(filter #(or@@ -402,7 +403,7 @@(fetch-compatible-seriescardoptions- {(:database_id card) (lib.metadata.jvm/application-database-metadata-provider (:database_id card))}+ {(:database_id card) (lib-be/application-database-metadata-provider (:database_id card))}[]))([card {:keys [page-size] :as options} database-id->metadata-provider current-cards]diff --git a/src/metabase/queries/models/card.clj b/src/metabase/queries/models/card.cljindex affcd0acf1e..24b9e9a4ffa 100644--- a/src/metabase/queries/models/card.clj+++ b/src/metabase/queries/models/card.clj@@ -18,6 +18,7 @@[metabase.dashboards.autoplace :as autoplace][metabase.events.core :as events][metabase.legacy-mbql.normalize :as mbql.normalize]+ [metabase.lib-be.core :as lib-be][metabase.lib-be.metadata.jvm :as lib.metadata.jvm][metabase.lib.core :as lib][metabase.lib.metadata.protocols :as lib.metadata.protocols]@@ -213,12 +214,7 @@(defn- source-card-id[query]- (when (map? query)- (let [query-type (lib/normalized-query-type query)]- (case query-type- :query (-> query mbql.normalize/normalize #_{:clj-kondo/ignore [:deprecated-var]} qp.util/query->source-card-id)- :mbql/query (-> query lib/normalize lib.util/source-card-id)- nil))))+ (some-> query not-empty lib-be/normalize-query lib/source-card-id))(defn- card->integer-table-ids"Return integer source table ids for card's :dataset_query."diff --git a/src/metabase/queries/models/query.clj b/src/metabase/queries/models/query.cljindex d6b21b406c8..0a50a6d7233 100644--- a/src/metabase/queries/models/query.clj+++ b/src/metabase/queries/models/query.clj@@ -148,7 +148,10 @@(parse-long card-id-str))))(defn collect-card-ids- "Return a sequence of model ids referenced in the MBQL `query`."+ "Return a sequence of model ids referenced in the MBQL `query`.++ DEPRECATED: Use [[metabase.lib.core/all-source-card-ids]] going forward."+ {:deprecated "0.57.0"}[query](let [ids (java.util.HashSet.)walker (fn [form]diff --git a/src/metabase/query_permissions/impl.clj b/src/metabase/query_permissions/impl.cljindex 742f3e9d896..d2fd8a95383 100644--- a/src/metabase/query_permissions/impl.clj+++ b/src/metabase/query_permissions/impl.clj@@ -9,6 +9,7 @@[clojure.walk :as walk][metabase.api.common :as api][metabase.legacy-mbql.normalize :as mbql.normalize]+ [metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.lib.metadata.protocols :as lib.metadata.protocols]@@ -19,7 +20,6 @@[metabase.query-processor.error-type :as qp.error-type];; legacy usage -- don't do things like this going forward^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.query-processor.store :as qp.store]- [metabase.query-processor.util :as qp.util][metabase.util :as u][metabase.util.i18n :refer [tru]][metabase.util.log :as log]@@ -214,45 +214,58 @@card-ids)})))(defn- legacy-mbql-required-perms- [query {:keys [throw-exceptions? already-preprocessed?]}]- (try- (let [query (mbql.normalize/normalize query)]- ;; if we are using a Card as our source, our perms are that Card's (i.e. that Card's Collection's) read perms- (if-let [source-card-id #_{:clj-kondo/ignore [:deprecated-var]} (qp.util/query->source-card-id query)]- {:paths (source-card-read-perms source-card-id)}- ;; otherwise if there's no source card then calculate perms based on the Tables referenced in the query- (let [query (cond-> query- (not already-preprocessed?) preprocess-query)- {:keys [table-ids table-query-ids card-ids native?]} (query->source-ids query)]- (merge- (when (seq card-ids)- {:card-ids card-ids})- (when (seq table-ids)- {:perms/view-data (zipmap table-ids (repeat :unrestricted))})- (when (seq table-query-ids)- {:perms/create-queries (zipmap table-query-ids (repeat :query-builder))})- (when native?- (native-query-perms query))))))- ;; if for some reason we can't expand the Card (i.e. it's an invalid legacy card) just return a set of permissions- ;; that means no one will ever get to see it- (catch Throwable e- (let [e (ex-info "Error calculating permissions for query"- {:query (or (u/ignore-exceptions (mbql.normalize/normalize query))- query)}- e)]- (if throw-exceptions? (throw e) (log/error e)))- {:perms/create-queries {0 :query-builder}}))) ; table 0 will never exist+ ([query options]+ (legacy-mbql-required-perms nil query options))++ ([metadata-provider+ query+ {:keys [throw-exceptions? already-preprocessed?]}]+ (try+ (let [metadata-provider (or metadata-provider+ (when (qp.store/initialized?)+ (qp.store/metadata-provider)))+ query (mbql.normalize/normalize query)]+ ;; if we are using a Card as our source, our perms are that Card's (i.e. that Card's Collection's) read perms+ (if-let [source-card-id (some-> query+ not-empty+ (->> (lib-be/normalize-query metadata-provider))+ lib/source-card-id)]+ {:paths (source-card-read-perms source-card-id)}+ ;; otherwise if there's no source card then calculate perms based on the Tables referenced in the query+ (let [query (cond-> query+ (not already-preprocessed?) preprocess-query)+ {:keys [table-ids table-query-ids card-ids native?]} (query->source-ids query)]+ (merge+ (when (seq card-ids)+ {:card-ids card-ids})+ (when (seq table-ids)+ {:perms/view-data (zipmap table-ids (repeat :unrestricted))})+ (when (seq table-query-ids)+ {:perms/create-queries (zipmap table-query-ids (repeat :query-builder))})+ (when native?+ (native-query-perms query))))))+ ;; if for some reason we can't expand the Card (i.e. it's an invalid legacy card) just return a set of permissions+ ;; that means no one will ever get to see it+ (catch Throwable e+ (let [e (ex-info "Error calculating permissions for query"+ {:query (or (u/ignore-exceptions (mbql.normalize/normalize query))+ query)}+ e)]+ (if throw-exceptions? (throw e) (log/error e)))+ {:perms/create-queries {0 :query-builder}})))) ; table 0 will never exist(defn- mbql5-required-perms"For MBQL 5 queries: for now, just convert it to legacy then hand off to thelegacy implementation(s) of [[required-perms]]."[query perms-opts]- (-> query- lib/normalize- ;; allowing for now until we convert this namespace to be MBQL-5-only- #_{:clj-kondo/ignore [:discouraged-var]}- lib/->legacy-MBQL- (legacy-mbql-required-perms perms-opts)))+ (let [mp (when (lib/metadata-provider? (:lib/metadata query))+ (:lib/metadata query))]+ (-> query+ lib/normalize+ ;; allowing for now until we convert this namespace to be MBQL-5-only+ #_{:clj-kondo/ignore [:discouraged-var]}+ lib/->legacy-MBQL+ (as-> $query (legacy-mbql-required-perms mp $query perms-opts)))))(defn required-perms-for-query"Returns a map representing the permissions requried to run `query`. The map has the optional keysdiff --git a/src/metabase/query_processor/api.clj b/src/metabase/query_processor/api.cljindex 61cfbb7880d..80b743fad4d 100644--- a/src/metabase/query_processor/api.clj+++ b/src/metabase/query_processor/api.clj@@ -6,10 +6,12 @@[metabase.driver :as driver][metabase.driver.util :as driver.u][metabase.events.core :as events]- ;; legacy usage -- don't use Legacy MBQL utils in QP code going forward, prefer Lib. This will be updated to use+ ;; legacy usage -- don't use Legacy MBQL utils in QP code going forward, prefer Lib. This will be updated to use;; Lib soon^{:clj-kondo/ignore [:discouraged-namespace]}[metabase.legacy-mbql.normalize :as mbql.normalize]+ [metabase.lib-be.core :as lib-be]+ [metabase.lib.core :as lib][metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.info :as lib.schema.info][metabase.model-persistence.core :as model-persistence]@@ -26,7 +28,6 @@[metabase.query-processor.pivot :as qp.pivot][metabase.query-processor.schema :as qp.schema][metabase.query-processor.streaming :as qp.streaming]- [metabase.query-processor.util :as qp.util][metabase.util :as u][metabase.util.i18n :refer [tru]][metabase.util.json :as json]@@ -45,8 +46,8 @@`:card-id` context can be passed along with the query so Collections perms checking is done if appropriate. This fnis a wrapper for the function of the same name in the QP util namespace; it adds additional permissions checking aswell."- [outer-query]- (when-let [source-card-id #_{:clj-kondo/ignore [:deprecated-var]} (qp.util/query->source-card-id outer-query)]+ [query]+ (when-let [source-card-id (some-> query not-empty lib-be/normalize-query lib/source-card-id)](log/infof "Source query for this query is Card %s" (pr-str source-card-id))(api/read-check :model/Card source-card-id)source-card-id))diff --git a/src/metabase/query_processor/streaming.clj b/src/metabase/query_processor/streaming.cljindex b36ac2feb75..65325fa1eb5 100644--- a/src/metabase/query_processor/streaming.clj+++ b/src/metabase/query_processor/streaming.clj@@ -49,14 +49,17 @@"Deduplicate column names that would otherwise conflict.TODO: This function includes logic that is normally is done by the annotate middleware, but hasn't been run yet- at this point in the code. We should eventually refactor this (#17195)"+ at this point in the code. We should eventually refactor this (#17195)++ TODO (Cam 9/23/25) -- We should use [[metabase.lib.field.util/add-deduplicated-names]] to do this."[cols](map (fn [col unique-name]- (let [col-with-display-name (if (:display_name col)- col- (assoc col :display_name (:name col)))]- (assoc col-with-display-name :name unique-name)))+ (-> col+ (cond-> (not (:display_name col)) (assoc :display_name (:name col)))+ (assoc :name unique-name)))cols+ ;; existing usage -- do not use going forward+ #_{:clj-kondo/ignore [:deprecated-var]}(mbql.u/uniquify-names (map :name cols))))(defn- validate-table-columnsdiff --git a/src/metabase/query_processor/util.clj b/src/metabase/query_processor/util.cljindex 11d95bd3917..4c5a59e34d0 100644--- a/src/metabase/query_processor/util.clj+++ b/src/metabase/query_processor/util.clj@@ -144,17 +144,6 @@e))))](buddy-hash/sha3-256 (json/encode (select-keys-for-hashing query)))))-;;; --------------------------------------------- Query Source Card IDs ------------------------------------------------(defn query->source-card-id- "Return the ID of the Card used as the \"source\" query of this query, if applicable; otherwise return `nil`."- {:deprecated "0.57.0"}- ^Integer [outer-query]- (let [source-table (get-in outer-query [:query :source-table])]- (when (string? source-table)- (when-let [[_ card-id-str] (re-matches #"^card__(\d+$)" source-table)]- (Integer/parseInt card-id-str)))))-;;; ------------------------------------------- Metadata Combination Utils --------------------------------------------(defn field-ref->keydiff --git a/src/metabase/segments/models/segment.clj b/src/metabase/segments/models/segment.cljindex 9ba1ce33f33..4aced0ea630 100644--- a/src/metabase/segments/models/segment.clj+++ b/src/metabase/segments/models/segment.clj@@ -114,7 +114,8 @@(lib.metadata.protocols/store-metadatas!(map #(lib.metadata.jvm/instance->metadata % :metadata/segment)segments)))- field-ids (mbql.u/referenced-field-ids (map :definition segments))+ ;; existing usage, do not use going forward+ field-ids #_{:clj-kondo/ignore [:deprecated-var]} (mbql.u/referenced-field-ids (map :definition segments))fields (when (seq field-ids)(lib.metadata.protocols/metadatas metadata-provider {:lib/type :metadata/column, :id (set field-ids)}))table-ids (into #{}diff --git a/src/metabase/xrays/automagic_dashboards/combination.clj b/src/metabase/xrays/automagic_dashboards/combination.cljindex f254788014c..ebbfac02b50 100644--- a/src/metabase/xrays/automagic_dashboards/combination.clj+++ b/src/metabase/xrays/automagic_dashboards/combination.clj@@ -21,6 +21,8 @@[clojure.walk :as walk][medley.core :as m][metabase.driver.util :as driver.u]+ [metabase.legacy-mbql.normalize :as mbql.normalize]+ [metabase.lib.schema.id :as lib.schema.id][metabase.models.interface :as mi][metabase.queries.core :as queries][metabase.query-processor.util :as qp.util]@@ -75,23 +77,25 @@#{:type/CreationTimestamp} :fail}#{:type/Integer}))-(defn add-dataset-query+(mu/defn- add-dataset-query :- [:map+ [:dataset_query ::ads/query]]"Add the `:dataset_query` key to this metric. Requires both the current metric-definition (from the grounded metric)and the database and table ids (from the source object)."[{:keys [metric-definition] :as ground-metric-with-dimensions}- {{:keys [database]} :root :keys [source query-filter]}]+ {{:keys [database]} :root :keys [source query-filter]} :- ::ads/context](let [source-table (if (->> source (mi/instance-of? :model/Table))(-> source u/the-id)(->> source u/the-id (str "card__")))model? (and (mi/instance-of? :model/Card source)(queries/model? source))](assoc ground-metric-with-dimensions- :dataset_query {:database database- :type :query- :query (cond-> (assoc metric-definition- :source-table source-table)- (and (not model?)- query-filter) (assoc :filter query-filter))})))+ :dataset_query (-> {:database database+ :type :query+ :query (cond-> (assoc metric-definition+ :source-table source-table)+ (and (not model?)+ query-filter) (assoc :filter query-filter))}+ mbql.normalize/normalize))))(defn- instantiate-visualization[[k v] dimensions metrics]@@ -186,10 +190,16 @@(complement (comp cell-dimension? id-or-name)))))));;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(mu/defn grounded-metrics->dashcards :- [:sequential ads/combined-metric]+(mu/defn grounded-metrics->dashcards :- [:sequential+ [:merge+ ads/combined-metric+ [:map+ [:dataset_query+ [:map+ [:database ::lib.schema.id/database]]]]]]"Generate dashcards from ground dimensions, using the base context, ground dimensions,card templates, and grounded metrics as input."- [base-context+ [base-context :- ::ads/contextcard-templatesground-dimensions :- ads/dim-name->matching-fieldsground-filters@@ -242,7 +252,7 @@:affinity-name card-name:card-score card-score:total-score (long (/ (apply + score-components) (count score-components)))- ;; Update dimension-name->field to include named contributions from both metrics and dimensions+ ;; Update dimension-name->field to include named contributions from both metrics and dimensions:dimension-name->field all-names->field:score-components score-components)(update :metric-definition add-aggregations final-aggregate)diff --git a/src/metabase/xrays/automagic_dashboards/comparison.clj b/src/metabase/xrays/automagic_dashboards/comparison.cljindex 2456cfec01e..e0bec2b801a 100644--- a/src/metabase/xrays/automagic_dashboards/comparison.clj+++ b/src/metabase/xrays/automagic_dashboards/comparison.clj@@ -3,15 +3,13 @@[medley.core :as m][metabase.api.common :as api][metabase.legacy-mbql.normalize :as mbql.normalize]+ [metabase.lib.schema.id :as lib.schema.id][metabase.models.interface :as mi][metabase.query-processor.util :as qp.util][metabase.util :as u][metabase.util.i18n :refer [tru]]- [metabase.xrays.automagic-dashboards.core- :refer [->related-entity- ->root- automagic-analysis- capitalize-first]]+ [metabase.util.malli :as mu]+ [metabase.xrays.automagic-dashboards.core :refer [->related-entity ->root automagic-analysis capitalize-first]][metabase.xrays.automagic-dashboards.filters :as filters][metabase.xrays.automagic-dashboards.names :as names][metabase.xrays.automagic-dashboards.populate :as populate]@@ -191,8 +189,11 @@distinct(map (partial magic.util/->field segment))))-(defn- update-related- [related left right]+(mu/defn- update-related+ [related+ left :- [:map+ [:database ::lib.schema.id/database]]+ right](-> related(update :related (comp distinct conj) (-> right :entity ->related-entity))(assoc :compare (concatdiff --git a/src/metabase/xrays/automagic_dashboards/core.clj b/src/metabase/xrays/automagic_dashboards/core.cljindex 976c4c6b882..69eb954ad54 100644--- a/src/metabase/xrays/automagic_dashboards/core.clj+++ b/src/metabase/xrays/automagic_dashboards/core.clj@@ -149,6 +149,10 @@[medley.core :as m][metabase.analyze.core :as analyze][metabase.legacy-mbql.normalize :as mbql.normalize]+ [metabase.lib-be.core :as lib-be]+ [metabase.lib.core :as lib]+ [metabase.lib.schema :as lib.schema]+ [metabase.lib.schema.id :as lib.schema.id][metabase.models.interface :as mi][metabase.query-processor.util :as qp.util][metabase.util :as u]@@ -163,6 +167,7 @@[metabase.xrays.automagic-dashboards.interesting :as interesting][metabase.xrays.automagic-dashboards.names :as names][metabase.xrays.automagic-dashboards.populate :as populate]+ [metabase.xrays.automagic-dashboards.schema :as ads][metabase.xrays.automagic-dashboards.util :as magic.util][metabase.xrays.related :as related][toucan2.core :as t2]))@@ -179,7 +184,7 @@{:arglists '([entity])}mi/model)-(defmethod ->root :model/Table+(mu/defmethod ->root :model/Table :- ::ads/root[table]{:entity table:full-name (:display_name table)@@ -189,7 +194,7 @@:url (format "%stable/%s" public-endpoint (u/the-id table)):dashboard-templates-prefix ["table"]})-(defmethod ->root :model/Segment+(mu/defmethod ->root :model/Segment :- ::ads/root[segment](let [table (->> segment :table_id (t2/select-one :model/Table :id))]{:entity segment@@ -202,37 +207,43 @@:url (format "%ssegment/%s" public-endpoint (u/the-id segment)):dashboard-templates-prefix ["table"]}))-(defmethod ->root :xrays/Metric- [metric]- (let [table (->> metric :table_id (t2/select-one :model/Table :id))]+(mu/defmethod ->root :xrays/Metric :- ::ads/root+ [metric :- ::ads/metric]+ (let [table (some->> metric :table_id (t2/select-one :model/Table :id))]{:entity metric:full-name (if (:id metric)(trun "{0} metric" "{0} metrics" (:name metric))(:name metric)):short-name (:name metric):source table- :database (:db_id table)+ :database (or (:db_id table)+ (:xrays/database-id metric));; We use :id here as it might not be a concrete field but rather one from a nested query which;; does not have an ID.:url (format "%smetric/%s" public-endpoint (:id metric)):dashboard-templates-prefix ["metric"]}))-(defmethod ->root :model/Field- [field]+(mu/defmethod ->root :model/Field :- ::ads/root+ [field :- ::ads/field](let [table (field/table field)]{:entity field:full-name (trun "{0} field" "{0} fields" (:display_name field)):short-name (:display_name field):source table- :database (:db_id table)+ :database (or (:db_id table) (:xrays/database-id field));; We use :id here as it might not be a concrete metric but rather one from a nested query;; which does not have an ID.:url (format "%sfield/%s" public-endpoint (:id field)):dashboard-templates-prefix ["field"]}))-(def ^:private ^{:arglists '([card-or-question])} nested-query?+(defn- source-card-id [card-or-question]+ (binding [lib.schema/*HACK-disable-join-alias-in-field-ref-validation* true]+ (some-> card-or-question :dataset_query not-empty lib-be/normalize-query lib/source-card-id)))++(defn- nested-query?"Is this card or question derived from another model or question?"- (comp some? #_{:clj-kondo/ignore [:deprecated-var]} qp.util/query->source-card-id :dataset_query))+ [card-or-question]+ (some? (source-card-id card-or-question)))(def ^:private ^{:arglists '([card-or-question])} native-query?"Is this card or question native (SQL)?"@@ -240,7 +251,7 @@(defn- source-question[card-or-question]- (when-let [source-card-id #_{:clj-kondo/ignore [:deprecated-var]} (qp.util/query->source-card-id (:dataset_query card-or-question))]+ (when-let [source-card-id (source-card-id card-or-question)](t2/select-one :model/Card :id source-card-id)))(defn- table-like?@@ -272,7 +283,7 @@(native-query? card) (-> card (assoc :entity_type :entity/GenericTable)):else (->> card table-id (t2/select-one :model/Table :id))))-(defmethod ->root :model/Card+(mu/defmethod ->root :model/Card :- ::ads/root[card](let [source (source card)]{:entity card@@ -286,18 +297,20 @@"table""question")]}))-(defmethod ->root :model/Query- [query]- (let [source (source query)]+(mu/defmethod ->root :model/Query :- ::ads/root+ [query :- [:map+ [:database-id ::lib.schema.id/database]]]+ (let [source (source query)+ root {:database (:database-id query), :source source}]{:entity query:source source:database (:database-id query):query-filter (get-in query [:dataset_query :query :filter]):full-name (cond(native-query? query) (tru "Native query")- (table-like? query) (-> source ->root :full-name)- :else (names/question-description {:source source} query))- :short-name (names/source-name {:source source})+ (table-like? query) (-> source ->root :full-name)+ :else (names/question-description root query))+ :short-name (names/source-name root):url (format "%sadhoc/%s" public-endpoint(magic.util/encode-base64-json (:dataset_query query))):dashboard-templates-prefix [(if (table-like? query)@@ -392,8 +405,11 @@:when (some-> target mi/can-read?)](-> target field/table (assoc :link id))))-(def ^:private ^{:arglists '([source])} source->db- (comp (partial t2/select-one :model/Database :id) (some-fn :db_id :database_id)))+(defn- source->db [source]+ (let [db-id (or ((some-fn :db_id :database_id) source)+ (throw (ex-info "Source is missing Database ID"+ {:source source})))]+ (t2/select-one :model/Database :id db-id)))(defn- relevant-fields"Source fields from tables that are applicable to the entity being x-rayed."@@ -423,11 +439,11 @@(constantly source-fields))(constantly [])))))-(defn- make-base-context+(mu/defn- make-base-context :- ::ads/context"Create the underlying context to which we will add metrics, dimensions, and filters.This is applicable to all dashboard templates."- [{:keys [source] :as root}]+ [{:keys [source] :as root} :- ::ads/root]{:pre [source]}(let [tables (concat [source] (when (mi/instance-of? :model/Table source)(linked-tables source)))@@ -450,12 +466,12 @@(assoc group :title (or comparison_title title))))))(instantiate-metadata context available-metrics {}))))-(defn- generate-base-dashboard+(mu/defn- generate-base-dashboard"Produce the \"base\" dashboard from the base context for an item and a dashboard template.This includes dashcards and global filters, but does not include related items and is not yet populated.Repeated calls of this might be generated (e.g. the main dashboard and related) then combined once usingcreate dashboard."- [{root :root :as base-context}+ [{root :root :as base-context} :- ::ads/context{template-cards :cards:keys [dashboard_filters]:as dashboard-template}@@ -539,8 +555,9 @@(map ->related-entity)(hash-map :drilldown-fields))))-(defn- comparisons- [root]+(mu/defn- comparisons+ [root :- [:map+ [:database ::lib.schema.id/database]]]{:compare (concat(for [segment (->> root :entity related/related :segments (map ->root))]{:url (str (:url root) "/compare/segment/" (-> segment :entity u/the-id))@@ -658,9 +675,9 @@(comparisons root))(fill-related max-related (get related-selectors (-> root :entity mi/model)))))-(defn- generate-dashboard+(mu/defn- generate-dashboard"Produce a fully-populated dashboard from the base context for an item and a dashboard template."- [{{:keys [show url query-filter] :as root} :root :as base-context}+ [{{:keys [show url query-filter] :as root} :root :as base-context} :- ::ads/context{:as dashboard-template}{grounded-dimensions :dimensions :as grounded-values}](let [show (or show max-cards)@@ -679,9 +696,9 @@:auto_apply_filters true:width "fixed"))))-(defn- automagic-dashboard+(mu/defn- automagic-dashboard"Create dashboards for table `root` using the best matching heuristics."- [{:keys [dashboard-template dashboard-templates-prefix] :as root}]+ [{:keys [dashboard-template dashboard-templates-prefix] :as root} :- ::ads/root](let [base-context (make-base-context root){template-dimensions :dimensionstemplate-metrics :metrics@@ -718,8 +735,9 @@[segment opts](automagic-dashboard (merge (->root segment) opts)))-(defmethod automagic-analysis :xrays/Metric- [metric opts]+(mu/defmethod automagic-analysis :xrays/Metric+ [metric :- ::ads/metric+ opts](automagic-dashboard (merge (->root metric) opts)))(mu/defn- collect-metrics :- [:maybe [:sequential (ms/InstanceOf :xrays/Metric)]]@@ -747,14 +765,18 @@(= (:table_id field) (table-id question)))]field))-(defn- decompose-question- [root question opts]+(mu/defn- decompose-question+ [root :- ::ads/root+ question+ opts](letfn [(analyze [x](try- (automagic-analysis x (assoc opts- :source (:source root)- :query-filter (:query-filter root)- :database (:database root)))+ (automagic-analysis+ (assoc x :xrays/database-id (:database root))+ (assoc opts+ :source (:source root)+ :query-filter (:query-filter root)+ :database (:database root)))(catch Throwable e(throw (ex-info (tru "Error decomposing question: {0}" (ex-message e)){:root root, :question question, :object x}@@ -830,8 +852,9 @@{:cell-query cell-query:cell-url cell-url}))))-(defmethod automagic-analysis :model/Field- [field opts]+(mu/defmethod automagic-analysis :model/Field+ [field :- ::ads/field+ opts](automagic-dashboard (merge (->root field) opts)))(defn- load-tables-with-enhanced-table-statsdiff --git a/src/metabase/xrays/automagic_dashboards/names.clj b/src/metabase/xrays/automagic_dashboards/names.cljindex 1ac45be4b2d..772c907e489 100644--- a/src/metabase/xrays/automagic_dashboards/names.clj+++ b/src/metabase/xrays/automagic_dashboards/names.clj@@ -7,7 +7,9 @@[metabase.query-processor.util :as qp.util][metabase.util.date-2 :as u.date][metabase.util.i18n :refer [deferred-tru tru]]+ [metabase.util.malli :as mu][metabase.util.time :as u.time]+ [metabase.xrays.automagic-dashboards.schema :as ads][metabase.xrays.automagic-dashboards.util :as magic.util]));; TODO - rename "minumum" to "minimum". Note that there are internationalization string implications@@ -56,24 +58,26 @@(source-name root)))(metric-name metric)))))-(defn question-description+(mu/defn question-description"Generate a description for the question."- [root question]+ [root :- ::ads/root+ question :- [:map+ [:dataset_query ::ads/query]]](let [aggregations (->> (get-in question [:dataset_query :query :aggregation])(metric->description root))dimensions (->> (get-in question [:dataset_query :query :breakout])(mapcat magic.util/collect-field-references)- (map (comp :display_name- (partial magic.util/->field root)))+ (map (partial magic.util/->field root))+ (map :display_name)join-enumeration)](if dimensions(tru "{0} by {1}" aggregations dimensions)aggregations)))(defmulti ^:private humanize-filter-value- {:arglists '([fieldset [op & args]])}- (fn [_ [op & _args]]- (qp.util/normalize-token op)))+ {:arglists '([root mbql-clause])}+ (fn [_root [tag]]+ (qp.util/normalize-token tag)))(def ^:private unit-name (comp {:minute-of-hour (deferred-tru "minute"):hour-of-day (deferred-tru "hour")@@ -86,9 +90,10 @@:year (deferred-tru "year")}qp.util/normalize-token))-(defn item-reference->field+(mu/defn- item-reference->field"Turn a field reference into a field."- [root [item-type :as item-reference]]+ [root :- ::ads/root+ [item-type :as item-reference]](case item-type(:field "field") (let [normalized-field-reference (mbql.normalize/normalize item-reference)temporal-unit (lib.util.match/match-one normalized-field-reference@@ -118,7 +123,7 @@(cond->> display_name(some-> unit u.date/extract-units) (tru "{0} of {1}" (unit-name unit)))))-(defn pluralize+(defn- pluralize"Add appropriate pluralization suffixes for integer numbers."[x];; the `int` cast here is to fix performance warnings if `*warn-on-reflection*` is enabled@@ -128,7 +133,7 @@3 (tru "{0}rd" x)(tru "{0}th" x)))-(defn humanize-datetime+(defn- humanize-datetime"Convert a time data type into a human friendly string."[t unit](let [dt (if (integer? t)@@ -156,59 +161,68 @@:day-of-year:week-of-year) (u.date/extract dt unit))))-(defmethod humanize-filter-value :=- [root [_ field-reference value]]+(mu/defmethod humanize-filter-value :=+ [root :- ::ads/root+ [_ field-reference value]](let [{:keys [item-name effective_type base_type unit]} (item-reference->field root field-reference)](if (isa? (or effective_type base_type) :type/Temporal)(tru "{0} is {1}" item-name (humanize-datetime value unit))(tru "{0} is {1}" item-name value))))-(defmethod humanize-filter-value :>=- [root [_ field-reference value]]+(mu/defmethod humanize-filter-value :>=+ [root :- ::ads/root+ [_ field-reference value]](let [{:keys [item-name effective_type base_type unit]} (item-reference->field root field-reference)](if (isa? (or effective_type base_type) :type/Temporal)(tru "{0} is not before {1}" item-name (humanize-datetime value unit))(tru "{0} is at least {1}" item-name value))))-(defmethod humanize-filter-value :>- [root [_ field-reference value]]+(mu/defmethod humanize-filter-value :>+ [root :- ::ads/root+ [_ field-reference value]](let [{:keys [item-name effective_type base_type unit]} (item-reference->field root field-reference)](if (isa? (or effective_type base_type) :type/Temporal)(tru "{0} is after {1}" item-name (humanize-datetime value unit))(tru "{0} is greater than {1}" item-name value))))-(defmethod humanize-filter-value :<=- [root [_ field-reference value]]+(mu/defmethod humanize-filter-value :<=+ [root :- ::ads/root+ [_ field-reference value]](let [{:keys [item-name effective_type base_type unit]} (item-reference->field root field-reference)](if (isa? (or effective_type base_type) :type/Temporal)(tru "{0} is not after {1}" item-name (humanize-datetime value unit))(tru "{0} is no more than {1}" item-name value))))-(defmethod humanize-filter-value :<- [root [_ field-reference value]]+(mu/defmethod humanize-filter-value :<+ [root :- ::ads/root+ [_ field-reference value]](let [{:keys [item-name effective_type base_type unit]} (item-reference->field root field-reference)](if (isa? (or effective_type base_type) :type/Temporal)(tru "{0} is before {1}" item-name (humanize-datetime value unit))(tru "{0} is less than {1}" item-name value))))-(defmethod humanize-filter-value :between- [root [_ field-reference min-value max-value]]+(mu/defmethod humanize-filter-value :between+ [root :- ::ads/root+ [_ field-reference min-value max-value]](tru "{0} is between {1} and {2}" (item-name root field-reference) min-value max-value))-(defmethod humanize-filter-value :inside- [root [_ lat-reference lon-reference lat-max lon-min lat-min lon-max]]+(mu/defmethod humanize-filter-value :inside+ [root :- ::ads/root+ [_ lat-reference lon-reference lat-max lon-min lat-min lon-max]](tru "{0} is between {1} and {2}; and {3} is between {4} and {5}"(item-name root lon-reference) lon-min lon-max(item-name root lat-reference) lat-min lat-max))-(defmethod humanize-filter-value :and- [root [_ & clauses]]+(mu/defmethod humanize-filter-value :and+ [root :- ::ads/root+ [_ & clauses]](->> clauses(map (partial humanize-filter-value root))join-enumeration))-(defmethod humanize-filter-value :default- [root [_ field-reference value]]+(mu/defmethod humanize-filter-value :default+ [root :- ::ads/root+ [_ field-reference value]](let [{:keys [item-name effective_type base_type unit]} (item-reference->field root field-reference)](if (isa? (or effective_type base_type) :type/Temporal)(tru "{0} relates to {1}" item-name (humanize-datetime value unit))diff --git a/src/metabase/xrays/automagic_dashboards/populate.clj b/src/metabase/xrays/automagic_dashboards/populate.cljindex 09b4f5d80a5..e5a1e862285 100644--- a/src/metabase/xrays/automagic_dashboards/populate.clj+++ b/src/metabase/xrays/automagic_dashboards/populate.clj@@ -5,9 +5,12 @@[medley.core :as m][metabase.api.common :as api][metabase.appearance.core :as appearance]+ [metabase.lib.schema :as lib.schema]+ [metabase.lib.schema.id :as lib.schema.id][metabase.queries.core :as queries][metabase.query-processor.util :as qp.util][metabase.util.log :as log]+ [metabase.util.malli :as mu][metabase.xrays.automagic-dashboards.filters :as filters][metabase.xrays.automagic-dashboards.util :as magic.util][toucan2.core :as t2]))@@ -129,9 +132,14 @@:dashboard_tab_id nil:visualization_settings {}})-(defn- add-card+(mu/defn- add-card"Add a card to dashboard `dashboard` at position [`x`, `y`]."- [dashboard {:keys [title description dataset_query width height id] :as card} [x y]]+ [dashboard+ {:keys [title description dataset_query width height id] :as card} :- [:map+ [:dataset_query+ [:map+ [:database ::lib.schema.id/database]]]]+ [x y]](let [card (-> {:creator_id api/*current-user-id*:dataset_query dataset_query:description description@@ -139,7 +147,8 @@:collection_id nil:id (or id (gensym))}(merge (visualization-settings card))- queries/populate-card-query-fields)]+ (as-> $card (binding [lib.schema/*HACK-disable-join-alias-in-field-ref-validation* true]+ (queries/populate-card-query-fields $card))))](update dashboard :dashcards conj(merge (card-defaults){:col y@@ -226,8 +235,12 @@(def ^:private ^Long ^:const group-heading-height 2)-(defn- add-group- [dashboard grid group cards]+(mu/defn- add-group+ [dashboard grid group cards :- [:sequential+ [:map+ [:dataset_query+ [:map+ [:database ::lib.schema.id/database]]]]]](let [start-row (bottom-row grid)start-row (cond-> start-rowgroup (+ group-heading-height))]@@ -286,10 +299,17 @@(let [g (group-by f coll)](access key-order g))))-(defn create-dashboard+(mu/defn create-dashboard"Create dashboard and populate it with cards."([dashboard] (create-dashboard dashboard :all))- ([{:keys [title transient_title description groups filters cards]} n]+ ([{:keys [title transient_title description groups filters cards]} :- [:map+ [:cards+ [:sequential+ [:map+ [:dataset_query+ [:map+ [:database ::lib.schema.id/database]]]]]]]+ n](let [n (cond(= n :all) (count cards)(keyword? n) (Integer/parseInt (name n))@@ -318,7 +338,8 @@(fn [dashcard card](m/assoc-some dashcard :card card))dashcards- (queries/with-can-run-adhoc-query cards)))))]+ (binding [lib.schema/*HACK-disable-join-alias-in-field-ref-validation* true]+ (queries/with-can-run-adhoc-query cards))))))](log/debugf "Adding %s cards to dashboard %s:\n%s"(count cards)diff --git a/src/metabase/xrays/automagic_dashboards/schema.clj b/src/metabase/xrays/automagic_dashboards/schema.cljindex 15bc3106083..38e389ca80c 100644--- a/src/metabase/xrays/automagic_dashboards/schema.clj+++ b/src/metabase/xrays/automagic_dashboards/schema.clj@@ -1,47 +1,65 @@(ns metabase.xrays.automagic-dashboards.schema(:require[malli.core :as mc]- [malli.util :as mut]))+ [malli.util :as mut]+ [metabase.legacy-mbql.schema :as mbql.s]+ [metabase.lib.schema.id :as lib.schema.id]+ [metabase.util.malli.registry :as mr]))-(def context+(mr/def ::root+ [:map+ [:database ::lib.schema.id/database]])++(mr/def ::context"The big ball of mud data object from which we generate x-rays"(mc/schema[:map- [:source any?]- [:root any?]- [:tables {:optional true} any?]+ [:source any?]+ [:root [:ref ::root]]+ [:tables {:optional true} any?][:query-filter {:optional true} any?]]))-(def dashcard- "The base unit thing we are trying to produce in x-rays"- ;; TODO - Beef these specs up, esp. the any?s- (mc/schema+(mr/def ::query+ [:ref ::mbql.s/Query])++(mr/def ::table-id-or-database-id+ [:and[:map- [:dataset_query {:optional true}+ [:table_id {:optional true} [:maybe ::lib.schema.id/table]]+ [:xrays/database-id {:optional true} [:maybe ::lib.schema.id/database]]]+ [:fn+ {:error/message "If instance does not have :table_id, it must have :xrays/database-id"}+ (some-fn :table_id :xrays/database-id)]])++(mr/def ::field+ [:ref ::table-id-or-database-id])++(mr/def ::metric+ [:ref ::table-id-or-database-id])++#_(def dashcard+ "The base unit thing we are trying to produce in x-rays"+ ;; TODO - Beef these specs up, esp. the any?s+ (mc/schema[:map- [:database {:optional true} [:maybe nat-int?]]- [:type :keyword]- [:query [:map- [:aggregation [:sequential any?]]- [:breakout {:optional true} [:sequential any?]]- [:source-table [:or :int :string]]]]]]- [:dimensions {:optional true} [:sequential string?]]- [:group {:optional true} string?]- [:height pos-int?]- [:metrics {:optional true} any?]- [:position {:optional true} nat-int?]- [:card-score {:optional true} number?]- [:total-score {:optional true} nat-int?]- [:metric-score {:optional true} nat-int?]- [:score-components {:optional true} [:sequential nat-int?]]- [:title {:optional true} string?]- [:visualization {:optional true} any?]- [:width pos-int?]- [:x_label {:optional true} string?]]))--(def dashcards- "A bunch of dashcards"- (mc/schema [:maybe [:sequential dashcard]]))+ [:dataset_query {:optional true} [:ref ::query]]+ [:dimensions {:optional true} [:sequential string?]]+ [:group {:optional true} string?]+ [:height pos-int?]+ [:metrics {:optional true} any?]+ [:position {:optional true} nat-int?]+ [:card-score {:optional true} number?]+ [:total-score {:optional true} nat-int?]+ [:metric-score {:optional true} nat-int?]+ [:score-components {:optional true} [:sequential nat-int?]]+ [:title {:optional true} string?]+ [:visualization {:optional true} any?]+ [:width pos-int?]+ [:x_label {:optional true} string?]]))++#_(def dashcards+ "A bunch of dashcards"+ (mc/schema [:maybe [:sequential dashcard]]))(def field-type"A dimension reference, as either a semantic type or entity type and semantic type."@@ -100,100 +118,100 @@[:string {:min 1}]filter-value]))-(def card-value- "A specification for the basic keys in the value of a card template."- (mc/schema- [:map- [:dimensions {:optional true} [:vector (mc/schema- [:map-of- {:min 1 :max 1}- [:string {:min 1}]- [:map- [:aggregation {:optional true} string?]]])]]- [:metrics {:optional true} [:vector string?]]- [:filters {:optional true} [:vector string?]]- [:card-score {:optional true} nat-int?]]))--(def card-template- "A specification for the basic keys in a card template."- (mc/schema- [:map-of- {:min 1 :max 1}- [:string {:min 1}]- card-value]))--(def dashboard-template- "A specification for the basic keys in a dashboard template."- (mc/schema- [:map- [:dimensions {:optional true} [:vector dimension-template]]- [:metrics {:optional true} [:vector metric-template]]- [:filters {:optional true} [:vector filter-template]]- [:cards {:optional true} [:vector card-template]]]))+#_(def card-value+ "A specification for the basic keys in the value of a card template."+ (mc/schema+ [:map+ [:dimensions {:optional true} [:vector (mc/schema+ [:map-of+ {:min 1 :max 1}+ [:string {:min 1}]+ [:map+ [:aggregation {:optional true} string?]]])]]+ [:metrics {:optional true} [:vector string?]]+ [:filters {:optional true} [:vector string?]]+ [:card-score {:optional true} nat-int?]]))++#_(def card-template+ "A specification for the basic keys in a card template."+ (mc/schema+ [:map-of+ {:min 1 :max 1}+ [:string {:min 1}]+ card-value]))++#_(def dashboard-template+ "A specification for the basic keys in a dashboard template."+ (mc/schema+ [:map+ [:dimensions {:optional true} [:vector dimension-template]]+ [:metrics {:optional true} [:vector metric-template]]+ [:filters {:optional true} [:vector filter-template]]+ [:cards {:optional true} [:vector card-template]]]));; Available values schema -- These are items for which fields have been successfully bound-(def available-values- "Specify the shape of things that are available after dimension to field matching for affinity matching"- (mc/schema- [:map- [:available-dimensions [:map-of [:string {:min 1}] any?]]- [:available-metrics [:map-of [:string {:min 1}] any?]]- [:available-filters {:optional true} [:map-of [:string {:min 1}] any?]]]))+#_(def available-values+ "Specify the shape of things that are available after dimension to field matching for affinity matching"+ (mc/schema+ [:map+ [:available-dimensions [:map-of [:string {:min 1}] any?]]+ [:available-metrics [:map-of [:string {:min 1}] any?]]+ [:available-filters {:optional true} [:map-of [:string {:min 1}] any?]]]));; Schemas for "affinity" functions as these can be particularly confusing-(def dimension-set- "A set of dimensions that belong together. This is the basic unity of affinity."- [:set string?])+#_(def dimension-set+ "A set of dimensions that belong together. This is the basic unity of affinity."+ [:set string?])-(def semantic-affinity-set- "A set of sematic types that belong together. This is the basic unity of semantic affinity."- [:set :keyword])+#_(def semantic-affinity-set+ "A set of sematic types that belong together. This is the basic unity of semantic affinity."+ [:set :keyword])-(def affinity- "A collection of things that go together. In this case, we're a bit specialized on+#_(def affinity+ "A collection of things that go together. In this case, we're a bit specialized oncard affinity, but the key element in the structure is `:base-dims`, which are aset of dimensions which, when satisfied, enable this affinity object."- (mc/schema- [:map- [:affinity-name :string]- [:affinity-set [:set :keyword]]- [:card-template card-value]- [:metric-constituent-names [:sequential :string]]- [:metric-field-types [:set :keyword]]- [:named-dimensions [:sequential :string]]- [:score {:optional true} nat-int?]]))--(def affinities- "A sequence of affinity objects."- (mc/schema- [:sequential affinity]))--(def affinity-old- "A collection of things that go together. In this case, we're a bit specialized on+ (mc/schema+ [:map+ [:affinity-name :string]+ [:affinity-set [:set :keyword]]+ [:card-template card-value]+ [:metric-constituent-names [:sequential :string]]+ [:metric-field-types [:set :keyword]]+ [:named-dimensions [:sequential :string]]+ [:score {:optional true} nat-int?]]))++#_(def affinities+ "A sequence of affinity objects."+ (mc/schema+ [:sequential affinity]))++#_(def affinity-old+ "A collection of things that go together. In this case, we're a bit specialized oncard affinity, but the key element in the structure is `:base-dims`, which are aset of dimensions which, when satisfied, enable this affinity object."- (mc/schema- [:map- [:dimensions {:optional true} [:vector string?]]- [:metrics {:optional true} [:vector string?]]- [:filters {:optional true} [:vector string?]]- [:score {:optional true} nat-int?]- [:affinity-name string?]- [:base-dims dimension-set]]))--(def affinities-old- "A sequence of affinity objects."- (mc/schema- [:sequential affinity-old]))--(def affinity-matches- "A map of named affinities to all dimension sets that are associated with this name."- (mc/schema- [:map-of- :string- [:vector dimension-set]]))+ (mc/schema+ [:map+ [:dimensions {:optional true} [:vector string?]]+ [:metrics {:optional true} [:vector string?]]+ [:filters {:optional true} [:vector string?]]+ [:score {:optional true} nat-int?]+ [:affinity-name string?]+ [:base-dims dimension-set]]))++#_(def affinities-old+ "A sequence of affinity objects."+ (mc/schema+ [:sequential affinity-old]))++#_(def affinity-matches+ "A map of named affinities to all dimension sets that are associated with this name."+ (mc/schema+ [:map-of+ :string+ [:vector dimension-set]]))(def item"A \"thing\" that we bind to, consisting, generally, of at least a name and id"@@ -222,15 +240,15 @@dim-name->dim-defdim-name->matching-fields))-(def dimension-map- "A map of dimension names to item satisfying that dimensions"- (mc/schema- [:map-of :string item]))+#_(def dimension-map+ "A map of dimension names to item satisfying that dimensions"+ (mc/schema+ [:map-of :string item]))-(def dimension-maps- "A sequence of dimension maps"- (mc/schema- [:sequential dimension-map]))+#_(def dimension-maps+ "A sequence of dimension maps"+ (mc/schema+ [:sequential dimension-map]))(def normalized-metric-template"A \"normalized\" metric template is a map containing the metric name as a key@@ -263,9 +281,9 @@[:aggregation [:sequential any?]][:breakout [:sequential any?]]]]])))-(comment- (require '[malli.generator :as mg])- (mg/sample dashboard-template)- (mg/sample affinities)- (mg/sample affinity-matches)- (mg/sample grounded-metric))+#_(comment+ (require '[malli.generator :as mg])+ (mg/sample dashboard-template)+ (mg/sample affinities)+ (mg/sample affinity-matches)+ (mg/sample grounded-metric))diff --git a/src/metabase/xrays/automagic_dashboards/util.clj b/src/metabase/xrays/automagic_dashboards/util.cljindex fd6c9fa1973..c103f15ab00 100644--- a/src/metabase/xrays/automagic_dashboards/util.clj+++ b/src/metabase/xrays/automagic_dashboards/util.clj@@ -14,6 +14,7 @@[metabase.util.log :as log][metabase.util.malli :as mu][metabase.util.malli.schema :as ms]+ [metabase.xrays.automagic-dashboards.schema :as ads][ring.util.codec :as codec][toucan2.core :as t2]))@@ -64,9 +65,11 @@[form](lib.util.match/match form :field &match))-(mu/defn ->field :- [:maybe (ms/InstanceOf :model/Field)]+(mu/defn ->field :- [:maybe [:and+ (ms/InstanceOf :model/Field)+ ::ads/field]]"Return `Field` instance for a given ID or name in the context of root."- [{{result-metadata :result_metadata} :source, :as root}+ [{{result-metadata :result_metadata} :source, :as root} :- ::ads/rootfield-id-or-name-or-clause :- [:or ms/PositiveInt ms/NonBlankString [:fn mbql.preds/Field?]]](let [id-or-name (if (sequential? field-id-or-name-or-clause)(field-reference->id field-id-or-name-or-clause)@@ -81,11 +84,12 @@(log/warn "Warning: Automagic analysis context is missing result metadata. Unable to resolve Fields by name."))(when-let [field (m/find-first #(= (:name %) id-or-name)result-metadata)]- (as-> field field- (update field :base_type keyword)- (update field :semantic_type keyword)- (mi/instance :model/Field field)- (analyze/run-classifiers field {}))))+ (-> field+ (update :base_type keyword)+ (update :semantic_type keyword)+ (->> (mi/instance :model/Field))+ (assoc :xrays/database-id (:database root))+ (analyze/run-classifiers {}))));; otherwise this isn't returning something, and that's probably an error. Log it.(log/warnf "Cannot resolve Field %s in automagic analysis context\n%s" field-id-or-name-or-clause (u/pprint-to-str root)))))diff --git a/src/metabase/xrays/transforms/core.clj b/src/metabase/xrays/transforms/core.cljindex 8a64b204239..07f3f15548b 100644--- a/src/metabase/xrays/transforms/core.clj+++ b/src/metabase/xrays/transforms/core.clj@@ -135,7 +135,9 @@(maybe-add-limit local-bindings step))query {:type :query:query inner-query- :database ((some-fn :db_id :database_id) source-entity)}]+ :database (or ((some-fn :db_id :database_id) source-entity)+ (throw (ex-info "Source entity is missing Database ID"+ {:source-entity source-entity})))}](assoc bindings name {:entity (tf.materialize/make-card-for-step! step query):dimensions (infer-resulting-dimensions local-bindings step query)})))diff --git a/src/metabase/xrays/transforms/dashboard.clj b/src/metabase/xrays/transforms/dashboard.cljindex 66d49dc4c55..6f929846c61 100644--- a/src/metabase/xrays/transforms/dashboard.clj+++ b/src/metabase/xrays/transforms/dashboard.clj@@ -2,7 +2,9 @@(:require[medley.core :as m][metabase.api.common :as api]+ [metabase.lib.schema.id :as lib.schema.id][metabase.util :as u]+ [metabase.util.malli :as mu][metabase.xrays.automagic-dashboards.populate :as populate][metabase.xrays.transforms.materialize :as tf.materialize][metabase.xrays.transforms.specs :refer [transform-specs]]@@ -33,8 +35,9 @@:position 0})))cards))-(defn- card-for-source-table- [table]+(mu/defn- card-for-source-table+ [table :- [:map+ [:db_id ::lib.schema.id/database]]]{:pre [(map? table)]}{:creator_id api/*current-user-id*:dataset_query {:type :querydiff --git a/test/metabase/analytics/stats_test.clj b/test/metabase/analytics/stats_test.cljindex 328254c59e8..7ddfe9e6897 100644--- a/test/metabase/analytics/stats_test.clj+++ b/test/metabase/analytics/stats_test.clj@@ -355,41 +355,44 @@(deftest question-metrics-test(testing "Returns metrics for cards using a single sql query"- (mt/with-temp [:model/User u {}- :model/Dashboard d {:creator_id (u/the-id u)}- :model/Card _ {}- :model/Card _ {:query_type "native"}- :model/Card _ {:query_type "gui"}- :model/Card _ {:public_uuid (str (random-uuid))}- :model/Card _ {:dashboard_id (u/the-id d)}- :model/Card _ {:enable_embedding true}- :model/Card _ {:enable_embedding true :public_uuid (str (random-uuid))- :dataset_query {:native {:template-tags {:param {:name "param" :display-name "Param" :type :number}}}}- :embedding_params {:category_name "locked" :name_category "disabled"}}- :model/Card _ {:enable_embedding true :public_uuid (str (random-uuid))- :dataset_query {:native {:template-tags {:param {:name "param" :display-name "Param" :type :string}}}}- :embedding_params {:category_name "enabled" :name_category "enabled"}}]- (testing "reported metrics for all app db types"- (is (malli= [:map- [:questions [:map- [:total [:= 8]]- [:native [:= 1]]- [:gui [:= 1]]- [:is_dashboard_question [:= 1]]]]- [:public [:map- [:total [:= 3]]]]- [:embedded [:map- [:total [:= 3]]]]]- (#'stats/question-metrics))))- (when (contains? #{:mysql :postgres} (mdb/db-type))- (testing "reports json column derived-metrics"- (let [reported (#'stats/question-metrics)]- (is (= 2 (get-in reported [:questions :with_params])))- (is (= 2 (get-in reported [:public :with_params])))- (is (= 2 (get-in reported [:embedded :with_params])))- (is (= 1 (get-in reported [:embedded :with_enabled_params])))- (is (= 1 (get-in reported [:embedded :with_locked_params])))- (is (= 1 (get-in reported [:embedded :with_disabled_params])))))))))+ (mt/with-empty-h2-app-db!+ (mt/with-temp [:model/User u {}+ :model/Dashboard d {:creator_id (u/the-id u)}+ :model/Card _ {}+ :model/Card _ {:query_type "native"}+ :model/Card _ {:query_type "gui"}+ :model/Card _ {:public_uuid (str (random-uuid))}+ :model/Card _ {:dashboard_id (u/the-id d)}+ :model/Card _ {:enable_embedding true}+ :model/Card _ {:enable_embedding true :public_uuid (str (random-uuid))+ :dataset_query {:database (mt/id)+ :type :native+ :native {:query "SELECT *"+ :template-tags {:param {:name "param" :display-name "Param" :type :number}}}}+ :embedding_params {:category_name "locked" :name_category "disabled"}}+ :model/Card _ {:enable_embedding true :public_uuid (str (random-uuid))+ :dataset_query {:database (mt/id)+ :type :native+ :native {:query "SELECT *"+ :template-tags {:param {:name "param" :display-name "Param" :type :text}}}}+ :embedding_params {:category_name "enabled" :name_category "enabled"}}]+ (testing "reported metrics for all app db types"+ (is (=? {:questions {:total 8+ :native 3+ :gui 1+ :is_dashboard_question 1}+ :public {:total 3}+ :embedded {:total 3}}+ (#'stats/question-metrics))))+ (when (contains? #{:mysql :postgres} (mdb/db-type))+ (testing "reports json column derived-metrics"+ (let [reported (#'stats/question-metrics)]+ (is (= 2 (get-in reported [:questions :with_params])))+ (is (= 2 (get-in reported [:public :with_params])))+ (is (= 2 (get-in reported [:embedded :with_params])))+ (is (= 1 (get-in reported [:embedded :with_enabled_params])))+ (is (= 1 (get-in reported [:embedded :with_locked_params])))+ (is (= 1 (get-in reported [:embedded :with_disabled_params]))))))))))(deftest internal-content-metrics-test(testing "Internal content doesn't contribute to stats"diff --git a/test/metabase/api_keys/api_test.clj b/test/metabase/api_keys/api_test.cljindex 641e0e45f74..90dbaa6f474 100644--- a/test/metabase/api_keys/api_test.clj+++ b/test/metabase/api_keys/api_test.clj@@ -122,7 +122,7 @@(deftest api-count-works(mt/with-empty-h2-app-db!- (is (zero? (mt/user-http-request :crowberto :get 200 "api-key/count")))+ (is ((fnil zero? -1) (mt/user-http-request :crowberto :get 200 "api-key/count")))(mt/with-temp [:model/ApiKey _ {::api-keys/unhashed-key (api-key/generate-key):name "my cool name":user_id (mt/user->id :crowberto)diff --git a/test/metabase/collections/api_test.clj b/test/metabase/collections/api_test.cljindex f92f4861676..8d3921d99ce 100644--- a/test/metabase/collections/api_test.clj+++ b/test/metabase/collections/api_test.clj@@ -26,7 +26,7 @@(set! *warn-on-reflection* true)-(use-fixtures :once (fixtures/initialize :test-users-personal-collections :row-lock))+(use-fixtures :once (fixtures/initialize :db :web-server :test-users :test-users-personal-collections :row-lock))(defmacro ^:private with-collection-hierarchy!"Totally-rad macro that creates a Collection hierarchy and grants the All Users group perms for all the Collections@@ -2073,7 +2073,7 @@(->> (:data (mt/user-http-request :crowberto :get 200 "collection/root/items"))(filter #(str/includes? (:name %) "Personal Collection"))))))))))-(deftest fetch-root-items-archived-test+(deftest ^:parallel fetch-root-items-archived-test(testing "GET /api/collection/root/items"(testing "Can we look for `archived` stuff with this endpoint?"(mt/with-temp [:model/Card card {:name "Business Card", :archived true}]@@ -2092,14 +2092,22 @@:data(results-matching {:name "Business Card", :model "card"}))))))))-(deftest fetch-root-items-fully-parameterized-can-be-false-test+(deftest ^:parallel fetch-root-items-fully-parameterized-can-be-false-test(testing "GET /api/collection/root/items"(testing "fully_parameterized of a card can be false"(mt/with-temp [:model/Card card {:name "Business Card"- :dataset_query {:native {:template-tags {:param0 {:default 0}- :param1 {:required false}- :param2 {:required false}}- :query "select {{param0}}, {{param1}} [[ , {{param2}} ]]"}}}]+ :dataset_query {:database (mt/id)+ :type :native+ :native {:template-tags {:param0 {:type :number+ :display-name "Param 0"+ :default 0}+ :param1 {:type :number+ :display-name "Param 1"+ :required false}+ :param2 {:type :number+ :display-name "Param 2"+ :required false}}+ :query "select {{param0}}, {{param1}} [[ , {{param2}} ]]"}}}](is (partial= [{:name "Business Card":entity_id (:entity_id card):model "card"@@ -2108,13 +2116,20 @@:data(results-matching {:name "Business Card", :model "card"}))))))))-(deftest fetch-root-items-fully-parameterized-field-filter-test+(deftest ^:parallel fetch-root-items-fully-parameterized-field-filter-test(testing "GET /api/collection/root/items"(testing "fully_parameterized is false even if a required field-filter parameter has no default"(mt/with-temp [:model/Card card {:name "Business Card"- :dataset_query {:native {:template-tags {:param0 {:default 0}- :param1 {:type "dimension", :required true}}- :query "select {{param0}}, {{param1}}"}}}]+ :dataset_query {:database (mt/id)+ :type :native+ :native {:template-tags {:param0 {:type :number+ :display-name "Param 0"+ :default 0}+ :param1 {:type "dimension"+ :display-name "Param 1"+ :required true+ :dimension [:field 1 nil]}}+ :query "select {{param0}}, {{param1}}"}}}](is (partial= [{:name "Business Card":entity_id (:entity_id card):model "card"@@ -2123,13 +2138,19 @@:data(results-matching {:name "Business Card", :model "card"}))))))))-(deftest fetch-root-items-fully-parameterized-optional-required-test+(deftest ^:parallel fetch-root-items-fully-parameterized-optional-required-test(testing "GET /api/collection/root/items"(testing "fully_parameterized is false even if an optional required parameter has no default"(mt/with-temp [:model/Card card {:name "Business Card"- :dataset_query {:native {:template-tags {:param0 {:default 0}- :param1 {:required true}}- :query "select {{param0}}, [[ , {{param1}} ]]"}}}]+ :dataset_query {:database (mt/id)+ :type :native+ :native {:template-tags {:param0 {:type :number+ :display-name "Param 0"+ :default 0}+ :param1 {:type :number+ :display-name "Param 1"+ :required true}}+ :query "select {{param0}}, [[ , {{param1}} ]]"}}}](is (partial= [{:name "Business Card":entity_id (:entity_id card):model "card"@@ -2138,11 +2159,13 @@:data(results-matching {:name "Business Card", :model "card"}))))))))-(deftest fetch-root-items-fully-parameterized-parsing-exception-test+(deftest ^:parallel fetch-root-items-fully-parameterized-parsing-exception-test(testing "GET /api/collection/root/items"(testing "fully_parameterized is true if invalid parameter syntax causes a parsing exception to be thrown"(mt/with-temp [:model/Card card {:name "Business Card"- :dataset_query {:native {:query "select [[]]"}}}]+ :dataset_query {:database (mt/id)+ :type :native+ :native {:query "select [[]]"}}}](is (partial= [{:name "Business Card":entity_id (:entity_id card):model "card"@@ -2151,15 +2174,26 @@:data(results-matching {:name "Business Card", :model "card"}))))))))-(deftest fetch-root-items-fully-parameterized-all-defaults-test+(deftest ^:parallel fetch-root-items-fully-parameterized-all-defaults-test(testing "GET /api/collection/root/items"(testing "fully_parameterized is true if all obligatory parameters have defaults"(mt/with-temp [:model/Card card {:name "Business Card"- :dataset_query {:native {:template-tags {:param0 {:required false, :default 0}- :param1 {:required true, :default 1}- :param2 {}- :param3 {:type "dimension"}}- :query "select {{param0}}, {{param1}} [[ , {{param2}} ]] from t {{param3}}"}}}]+ :dataset_query {:database (mt/id)+ :type :native+ :native {:template-tags {:param0 {:type :number+ :display-name "Param 0"+ :required false+ :default 0}+ :param1 {:type :number+ :display-name "Param 1"+ :required true+ :default 1}+ :param2 {:type :number+ :display-name "Param 2"}+ :param3 {:type "dimension"+ :dimension [:field (mt/id :venues :id) nil]+ :display-name "Param 3"}}+ :query "select {{param0}}, {{param1}} [[ , {{param2}} ]] from t {{param3}}"}}}](is (partial= [{:name "Business Card":entity_id (:entity_id card):model "card"@@ -2168,20 +2202,25 @@:data(results-matching {:name "Business Card", :model "card"}))))))))-(deftest fetch-root-items-fully-parameterized-snippet-test+(deftest ^:parallel fetch-root-items-fully-parameterized-snippet-test(testing "GET /api/collection/root/items"(testing "fully_parameterized using a snippet without parameters is true"(mt/with-temp [:model/NativeQuerySnippet snippet {:content "table":creator_id (mt/user->id :crowberto):name "snippet"}:model/Card card {:name "Business Card"- :dataset_query {:native {:template-tags {:param0 {:required false- :default 0}- :snippet {:name "snippet"- :type :snippet- :snippet-name "snippet"- :snippet-id (:id snippet)}}- :query "select {{param0}} from {{snippet}}"}}}]+ :dataset_query {:database (mt/id)+ :type :native+ :native {:template-tags {:param0 {:type :number+ :display-name "Param 0"+ :required false+ :default 0}+ :snippet {:name "snippet"+ :display-name "Snippet"+ :type :snippet+ :snippet-name "snippet"+ :snippet-id (:id snippet)}}+ :query "select {{param0}} from {{snippet}}"}}}](is (partial= [{:name "Business Card":entity_id (:entity_id card):model "card"@@ -2190,7 +2229,7 @@:data(results-matching {:name "Business Card", :model "card"}))))))))-(deftest fetch-root-items-fully-parameterized-card-reference-test+(deftest ^:parallel fetch-root-items-fully-parameterized-card-reference-test(testing "GET /api/collection/root/items"(testing "a card with only a reference to another card is considered fully parameterized (#25022)"(mt/with-temp [:model/Card card-1 {:dataset_query (mt/mbql-query venues)}]diff --git a/test/metabase/dashboards/api_test.clj b/test/metabase/dashboards/api_test.cljindex 3fff2857ac4..e53392ef641 100644--- a/test/metabase/dashboards/api_test.clj+++ b/test/metabase/dashboards/api_test.clj@@ -4735,9 +4735,8 @@{:id (mt/id :products):fields sequential?}{:id (mt/id :venues)}])- (sort-by :id- [{:id (str "card__" card-id-2)- :fields sequential?}]))+ [{:id (str "card__" card-id-2)+ :fields sequential?}]):cards [{:id link-card}]:databases [{:id (mt/id) :engine string?}]:dashboards [{:id link-dash}]}@@ -4748,6 +4747,7 @@(mt/with-temp[:model/Card {card-id-1 :id} {:dataset_query (mt/mbql-query products)}:model/Card {card-id-2 :id} {:dataset_query {:type :query+ :database (mt/id):query {:source-table (str "card__" card-id-1)}}}:model/Dashboard {dashboard-id :id} {}:model/DashboardCard _ {:card_id card-id-2diff --git a/test/metabase/embedding/api/embed_test.clj b/test/metabase/embedding/api/embed_test.cljindex c88d0774e60..ede53aea980 100644--- a/test/metabase/embedding/api/embed_test.clj+++ b/test/metabase/embedding/api/embed_test.clj@@ -139,10 +139,6 @@(is (= [{:col "Count"} {:col 100.0}]actual))))))-(defn dissoc-id-and-name [obj]- (cond-> obj- (map? obj) (dissoc :id :name)))-(def successful-card-info"Data that should be returned if `GET /api/embed/card/:token` completes successfully (minus `:id` and `:name`).This should only be the bare minimum amount of info needed to display the Card, leaving out other data we wouldn't@@ -167,9 +163,8 @@(deftest it-should-be-possible-to-use-this-endpoint-successfully-if-all-the-conditions-are-met(with-embedding-enabled-and-new-secret-key!(with-temp-card [card {:enable_embedding true}]- (is (= successful-card-info- (dissoc-id-and-name- (client/client :get 200 (card-url card))))))))+ (is (=? successful-card-info+ (client/client :get 200 (card-url card)))))))(deftest we-should-fail-when-attempting-to-use-an-expired-token(with-embedding-enabled-and-new-secret-key!@@ -211,7 +206,8 @@(with-temp-card [card {:enable_embedding true:dataset_query {:database (mt/id):type :native- :native {:template-tags {:a {:type "date", :name "a", :display_name "a" :id "a"}+ :native {:query "SELECT 1;"+ :template-tags {:a {:type "date", :name "a", :display_name "a" :id "a"}:b {:type "date", :name "b", :display_name "b" :id "b"}:c {:type "date", :name "c", :display_name "c" :id "c"}:d {:type "date", :name "d", :display_name "d" :id "d"}}}}@@ -571,12 +567,10 @@(deftest it-should-be-possible-to-call-this-endpoint-successfully(with-embedding-enabled-and-new-secret-key!(mt/with-temp [:model/Dashboard dash {:enable_embedding true}]- (is (= successful-dashboard-info- (dissoc-id-and-name- (client/client :get 200 (dashboard-url dash)))))- (is (= successful-dashboard-info- (dissoc-id-and-name- (client/client :get 200 (dashboard-url (:entity_id dash) dash))))))))+ (is (=? successful-dashboard-info+ (client/client :get 200 (dashboard-url dash))))+ (is (=? successful-dashboard-info+ (client/client :get 200 (dashboard-url (:entity_id dash) dash)))))))(deftest bad-dashboard-id-fails(with-embedding-enabled-and-new-secret-key!diff --git a/test/metabase/embedding/api/preview_embed_test.clj b/test/metabase/embedding/api/preview_embed_test.cljindex 95fcbe5a254..e691f46697e 100644--- a/test/metabase/embedding/api/preview_embed_test.clj+++ b/test/metabase/embedding/api/preview_embed_test.clj@@ -23,29 +23,25 @@(embed-test/with-embedding-enabled-and-new-secret-key!(embed-test/with-temp-card [card](testing "it should be possible to use this endpoint successfully if all the conditions are met"- (is (= embed-test/successful-card-info- (embed-test/dissoc-id-and-name- (mt/user-http-request :crowberto :get 200 (card-url card))))))-+ (is (=? embed-test/successful-card-info+ (mt/user-http-request :crowberto :get 200 (card-url card)))))(testing "if the user is not an admin this endpoint should fail"(is (= "You don't have permissions to do that."(mt/user-http-request :rasta :get 403 (card-url card)))))-(testing "check that the endpoint doesn't work if embedding isn't enabled"(mt/with-temporary-setting-values [enable-embedding-static false](is (= "Embedding is not enabled."(embed-test/with-temp-card [card](mt/user-http-request :crowberto :get 400 (card-url card)))))))-(testing "check that if embedding is enabled globally requests fail if they are signed with the wrong key"(is (= "Message seems corrupt or manipulated"(mt/user-http-request :crowberto :get 400 (embed-test/with-new-secret-key! (card-url card))))))-(testing "Check that only ENABLED params that ARE NOT PRESENT IN THE JWT come back"(embed-test/with-temp-card [card {:dataset_query{:database (mt/id):type :native- :native {:template-tags {:a {:type "date", :name "a", :display_name "a" :id "a"}+ :native {:query "SELECT 1;"+ :template-tags {:a {:type "date", :name "a", :display_name "a" :id "a"}:b {:type "date", :name "b", :display_name "b" :id "b"}:c {:type "date", :name "c", :display_name "c" :id "c"}:d {:type "date", :name "d", :display_name "d" :id "d"}}}}}]@@ -226,19 +222,15 @@(embed-test/with-embedding-enabled-and-new-secret-key!(mt/with-temp [:model/Dashboard dash](testing "it should be possible to call this endpoint successfully..."- (is (= embed-test/successful-dashboard-info- (embed-test/dissoc-id-and-name- (mt/user-http-request :crowberto :get 200 (dashboard-url dash))))))-+ (is (=? embed-test/successful-dashboard-info+ (mt/user-http-request :crowberto :get 200 (dashboard-url dash)))))(testing "...but if the user is not an admin this endpoint should fail"(is (= "You don't have permissions to do that."(mt/user-http-request :rasta :get 403 (dashboard-url dash)))))-(testing "check that the endpoint doesn't work if embedding isn't enabled"(mt/with-temporary-setting-values [enable-embedding-static false](is (= "Embedding is not enabled."(mt/user-http-request :crowberto :get 400 (dashboard-url dash))))))-(testing "check that if embedding is enabled globally requests fail if they are signed with the wrong key"(is (= "Message seems corrupt or manipulated"(mt/user-http-request :crowberto :get 400 (embed-test/with-new-secret-key! (dashboard-url dash))))))))))diff --git a/test/metabase/legacy_mbql/util_test.cljc b/test/metabase/legacy_mbql/util_test.cljcindex e54ce90389a..8b8b8597482 100644--- a/test/metabase/legacy_mbql/util_test.cljc+++ b/test/metabase/legacy_mbql/util_test.cljc@@ -788,18 +788,6 @@(t/is (= [:aggregation 0](mbql.u/update-field-options [:aggregation 0 {:b 2}] dissoc :b))))))-(t/deftest ^:parallel remove-namespaced-options-test- (t/are [clause expected] (= expected- (mbql.u/remove-namespaced-options clause))- [:field 1 {::namespaced true}] [:field 1 nil]- [:field 1 {::namespaced true, :a 1}] [:field 1 {:a 1}]- [:expression "wow"] [:expression "wow"]- [:expression "wow" {::namespaced true}] [:expression "wow"]- [:expression "wow" {::namespaced true, :a 1}] [:expression "wow" {:a 1}]- [:aggregation 0] [:aggregation 0]- [:aggregation 0 {::namespaced true}] [:aggregation 0]- [:aggregation 0 {::namespaced true, :a 1}] [:aggregation 0 {:a 1}]))-(t/deftest ^:parallel with-temporal-unit-test(t/is (= [:field 1 {:temporal-unit :day}](mbql.u/with-temporal-unit [:field 1 nil] :day)))diff --git a/test/metabase/lib/metadata/result_metadata_test.cljc b/test/metabase/lib/metadata/result_metadata_test.cljcindex 5c78efe64a4..d4cd64ec9f3 100644--- a/test/metabase/lib/metadata/result_metadata_test.cljc+++ b/test/metabase/lib/metadata/result_metadata_test.cljc@@ -1094,3 +1094,15 @@[:field "NAME" {}][:field "NAME_2" {}]](map :field-ref (#'result-metadata/deduplicate-field-refs cols)))))))++(deftest ^:parallel remove-namespaced-options-test+ (are [clause expected] (= expected+ (#'result-metadata/remove-namespaced-options clause))+ [:field 1 {::namespaced true}] [:field 1 nil]+ [:field 1 {::namespaced true, :a 1}] [:field 1 {:a 1}]+ [:expression "wow"] [:expression "wow"]+ [:expression "wow" {::namespaced true}] [:expression "wow"]+ [:expression "wow" {::namespaced true, :a 1}] [:expression "wow" {:a 1}]+ [:aggregation 0] [:aggregation 0]+ [:aggregation 0 {::namespaced true}] [:aggregation 0]+ [:aggregation 0 {::namespaced true, :a 1}] [:aggregation 0 {:a 1}]))diff --git a/test/metabase/public_sharing/api_test.clj b/test/metabase/public_sharing/api_test.cljindex 48d94aef6e9..35c8f6ea712 100644--- a/test/metabase/public_sharing/api_test.clj+++ b/test/metabase/public_sharing/api_test.clj@@ -188,7 +188,8 @@{:enable_embedding true:dataset_query {:database (mt/id):type :native- :native {:template-tags {:a {:type "date", :name "a", :display_name "a" :id "a" :default "A TAG"}+ :native {:query "SELECT 1"+ :template-tags {:a {:type "date", :name "a", :display_name "a" :id "a" :default "A TAG"}:b {:type "date", :name "b", :display_name "b" :id "b" :default "B TAG"}:c {:type "date", :name "c", :display_name "c" :id "c" :default "C TAG"}:d {:type "date", :name "d", :display_name "d" :id "d" :default "D TAG"}}}}diff --git a/test/metabase/queries/api/card_test.clj b/test/metabase/queries/api/card_test.cljindex 7911875f189..4d54d369fd7 100644--- a/test/metabase/queries/api/card_test.clj+++ b/test/metabase/queries/api/card_test.clj@@ -345,8 +345,10 @@:model/Segment {segment-id :id} {:table_id table-id}:model/Card {model-id :id :as model} {:name "Model":type :model- :dataset_query {:query {:source-table (mt/id :venues)- :filter [:= [:field 1 nil] "1"]}}}+ :dataset_query {:database (mt/id)+ :type :query+ :query {:source-table (mt/id :venues)+ :filter [:= [:field 1 nil] "1"]}}};; matching question:model/Card card-1 {:name "Card 1":dataset_query {:query {:source-table (str "card__" model-id)@@ -385,11 +387,11 @@{:query (format "select o.id from orders o join {{%s}} q1 on o.PRODUCT_ID = q1.PRODUCT_ID"model-ref):template-tags {model-ref- {:id "2185b98b-20b3-65e6-8623-4fb56acb0ca7"- :name model-ref+ {:id "2185b98b-20b3-65e6-8623-4fb56acb0ca7"+ :name model-ref:display-name model-ref- :type :card- :card-id model-id}}})+ :type :card+ :card-id model-id}}}):database (mt/id)}};; native query reference doesn't match:model/Card card-5 {:name "Card 5"@@ -400,23 +402,35 @@{:query (format "select o.id %s from orders o join {{%s}} q1 on o.PRODUCT_ID = q1.PRODUCT_ID"model-ref card-ref):template-tags {card-ref- {:id "2185b98b-20b3-65e6-8623-4fb56acb0ca7"- :name card-ref+ {:id "2185b98b-20b3-65e6-8623-4fb56acb0ca7"+ :name card-ref:display-name card-ref- :type :card- :card-id card-id}}})+ :type :card+ :card-id card-id}}}):database (mt/id)}}:model/Database {other-database-id :id} {}- ;; database doesn't quite match- :model/Card card-6 {:name "Card 6", :database_id other-database-id- :dataset_query {:query {:source-table (str "card__" model-id)}}}+ ;; database doesn't quite match, but when we save the Card we should correct it to the ID used by the+ ;; model+ :model/Card card-6 {:name "Card 6"+ :database_id other-database-id+ :dataset_query {:database other-database-id+ :type :query+ :query {:source-table (str "card__" model-id)}}};; same as matching question, but archived:model/Card card-7 {:name "Card 7":archived true- :dataset_query {:query {:source-table (str "card__" model-id)}}}]+ :dataset_query {:database (mt/id)+ :type :query+ :query {:source-table (str "card__" model-id)}}}]+ (testing "When saving Card 6, we should correct the database_id to the same ID used by the source card"+ (is (= (mt/id)+ (:database_id card-6))))(testing "list cards using a model"(with-cards-in-readable-collection! [model card-1 card-3 card-4 card-5 card-6 card-7]- (is (= #{"Card 1" "Card 3" "Card 4"}+ (is (= #{"Card 1"+ "Card 3"+ "Card 4"+ "Card 6"}(into #{} (map :name) (mt/user-http-request :rasta :get 200 "card":f :using_model :model_id model-id))))(is (= #{"Card 1" "Card 3"}@@ -2597,6 +2611,9 @@(testing "Changing core attributes un-verifies the card"(with-card :verified(is (verified? card))+ (is (=? {:dataset_query {:query {:source-table pos-int?}}}+ card)+ "(This test is written to expect a legacy query)")(update-card card (update-in card [:dataset_query :query :source-table] inc))(is (not (verified? card)))(testing "The unverification edit has explanatory text"@@ -3677,8 +3694,9 @@(testing "Don't throw an error if source card is deleted (#48461)"(mt/with-temp[:model/Card {card-id-1 :id} {:dataset_query (mt/mbql-query products)}- :model/Card {card-id-2 :id} {:dataset_query {:type :query- :query {:source-table (str "card__" card-id-1)}}}]+ :model/Card {card-id-2 :id} {:dataset_query {:type :query+ :database (mt/id)+ :query {:source-table (str "card__" card-id-1)}}}](letfn [(query-metadata [expected-status card-id](-> (mt/user-http-request :crowberto :get expected-status (str "card/" card-id "/query_metadata"))(api.test-util/select-query-metadata-keys-for-debugging)))]@@ -4308,7 +4326,7 @@(mt/with-temp [:model/Card {id-a :id} {:dataset_query (lib/->legacy-MBQL query-a) :type :question}](let [query-b (mt/native-query {:query "select * from {{#100-base-query}}":template-tags- {:#100-base-query+ {"#100-base-query"{:type :card:name "#100-base-query":id (random-uuid)diff --git a/test/metabase/queries/models/card_test.clj b/test/metabase/queries/models/card_test.cljindex e7e83c4654b..329a3bea7bb 100644--- a/test/metabase/queries/models/card_test.clj+++ b/test/metabase/queries/models/card_test.clj@@ -625,7 +625,9 @@(testing "cards which have another card as the source depend on that card"(mt/with-temp [:model/Card card1 {:name "base card"}:model/Card card2 {:name "derived card"- :dataset_query {:query {:source-table (str "card__" (:id card1))}}}]+ :dataset_query {:database (mt/id)+ :type :query+ :query {:source-table (str "card__" (:id card1))}}}](is (empty? (serdes/descendants "Card" (:id card1))))(is (= {["Card" (:id card1)] {"Card" (:id card2)}}(serdes/descendants "Card" (:id card2)))))))@@ -635,12 +637,14 @@(mt/with-temp [:model/NativeQuerySnippet snippet {:name "category" :content "category = 'Gizmo'"}:model/Card card{:name "Business Card"- :dataset_query {:native- {:template-tags {:snippet {:name "snippet"- :type :snippet- :snippet-name "snippet"- :snippet-id (:id snippet)}}- :query "select * from products where {{snippet}}"}}}]+ :dataset_query {:database (mt/id)+ :type :native+ :native {:template-tags {:snippet {:name "snippet"+ :display-name "Snippet"+ :type :snippet+ :snippet-name "snippet"+ :snippet-id (:id snippet)}}+ :query "select * from products where {{snippet}}"}}}](is (= {["NativeQuerySnippet" (:id snippet)] {"Card" (:id card)}}(serdes/descendants "Card" (:id card)))))))diff --git a/test/metabase/queries/models/query_test.clj b/test/metabase/queries/models/query_test.cljindex bc4047a471c..9a4fb429e67 100644--- a/test/metabase/queries/models/query_test.clj+++ b/test/metabase/queries/models/query_test.clj@@ -67,6 +67,7 @@:card-id 100}"tag_2" {:type :card:card-id 200}}}}]+ #_{:clj-kondo/ignore [:deprecated-var]}(is (= #{100 200}(set (query/collect-card-ids query))))))@@ -74,6 +75,7 @@(let [query {:database 1:type :query:query {:source-query {:source-table "card__1000"}}}]+ #_{:clj-kondo/ignore [:deprecated-var]}(is (= #{1000}(set (query/collect-card-ids query))))))@@ -88,6 +90,7 @@:display-name "Tag 2":type :card:card-id 200}}})]+ #_{:clj-kondo/ignore [:deprecated-var]}(is (= #{100 200}(set (query/collect-card-ids query))))))@@ -96,5 +99,6 @@meta/metadata-provider[(lib/query meta/metadata-provider (lib.metadata/table meta/metadata-provider (meta/id :venues)))])query-with-source-card (lib/query metadata-provider (lib.metadata/card metadata-provider 1))]+ #_{:clj-kondo/ignore [:deprecated-var]}(is (= #{1}(set (query/collect-card-ids query-with-source-card))))))diff --git a/test/metabase/query_processor/api_test.clj b/test/metabase/query_processor/api_test.cljindex 50254e1d373..8416ba96628 100644--- a/test/metabase/query_processor/api_test.clj+++ b/test/metabase/query_processor/api_test.clj@@ -1019,8 +1019,9 @@(testing "Don't throw an error if source card is deleted (#48461)"(mt/with-temp[:model/Card {card-id-1 :id} {:dataset_query (mt/mbql-query products)}- :model/Card {card-id-2 :id} {:dataset_query {:type :query- :query {:source-table (str "card__" card-id-1)}}}]+ :model/Card {card-id-2 :id} {:dataset_query {:database (mt/id)+ :type :query+ :query {:source-table (str "card__" card-id-1)}}}](letfn [(query-metadata [expected-status card-id](-> (mt/user-http-request :crowberto :post expected-status"dataset/query_metadata"diff --git a/test/metabase/revisions/models/revision_test.clj b/test/metabase/revisions/models/revision_test.cljindex ce6e2b9066e..f894698d052 100644--- a/test/metabase/revisions/models/revision_test.clj+++ b/test/metabase/revisions/models/revision_test.clj@@ -58,9 +58,14 @@(testing (str "make sure we call the appropriate post-select methods on `:object` when a revision comes out of the ""DB. This is especially important for things like Cards where we need to make sure query is ""normalized")- (is (= {:model "Card", :object {:card_schema queries/starting-card-schema-version, :dataset_query {:type :query}}}+ (is (= {:model "Card", :object {:card_schema queries/starting-card-schema-version+ :dataset_query {:database (mt/id)+ :type :query+ :query {:source-table (mt/id :venues)}}}}(mt/derecordize- (mi/do-after-select :model/Revision {:model "Card", :object {:dataset_query {:type "query"}}}))))))+ (mi/do-after-select :model/Revision {:model "Card", :object {:dataset_query {:database (mt/id)+ :type "query"+ :query {:source-table (mt/id :venues)}}}}))))));;; # Default diff-* implementationsdiff --git a/test/metabase/test/data/users.clj b/test/metabase/test/data/users.cljindex 6345ae229a9..e6d9d89266c 100644--- a/test/metabase/test/data/users.clj+++ b/test/metabase/test/data/users.clj@@ -176,25 +176,30 @@(locking tokens(reset! tokens {})))-(def ^:private ^:dynamic *retrying-authentication* false)-(defn- client-fn [the-client username & args]- (try- (apply the-client (username->token username) args)- (catch ExceptionInfo e- (let [{:keys [status-code]} (ex-data e)]- (when-not (= status-code 401)- (throw e))- ;; If we got a 401 unauthenticated clear the tokens cache + recur- ;;- ;; If we're already recursively retrying throw an Exception so we don't recurse forever.- (when *retrying-authentication*- (throw (ex-info (format "Failed to authenticate %s after two tries: %s" username (ex-message e))- {:user username}- e)))- (clear-cached-session-tokens!)- (binding [*retrying-authentication* true]- (apply client-fn the-client username args))))))+ (letfn [(thunk []+ (apply the-client (username->token username) args))+ (rethrow-when-not-401 [e]+ (when-not (= (:status-code (ex-data e)) 401)+ (throw e)))]+ (try+ (thunk)+ (catch ExceptionInfo e+ (rethrow-when-not-401 e)+ ;; first retry: get lock, then try again.+ (locking tokens+ (try+ (thunk)+ (catch ExceptionInfo e+ (rethrow-when-not-401 e)+ ;; second retry: clear cached session tokens, then try again one last time+ (clear-cached-session-tokens!)+ (try+ (thunk)+ (catch ExceptionInfo e+ (throw (ex-info (format "Failed to authenticate %s after three tries: %s" username (ex-message e))+ {:user username}+ e)))))))))))(defn- user-request[the-client user & args]diff --git a/test/metabase/warehouse_schema/api/table_test.clj b/test/metabase/warehouse_schema/api/table_test.cljindex 5596547dca5..83e6e01a259 100644--- a/test/metabase/warehouse_schema/api/table_test.clj+++ b/test/metabase/warehouse_schema/api/table_test.clj@@ -798,7 +798,8 @@(testing "GET /api/table/card__:id/query_metadata for deleted cards (#48461)"(mt/with-temp[:model/Card {card-id-1 :id} {:dataset_query (mt/mbql-query products)}- :model/Card {card-id-2 :id} {:dataset_query {:type :query+ :model/Card {card-id-2 :id} {:dataset_query {:database (mt/id)+ :type :query:query {:source-table (str "card__" card-id-1)}}}](letfn [(query-metadata [expected-status card-id](->> (format "table/card__%d/query_metadata" card-id)@@ -867,7 +868,7 @@:type :model:dataset_query (mt/mbql-query venues)}](let [card-virtual-table-id (str "card__" (:id model))- metric-query {:database 2+ metric-query {:database (mt/id):type "query":query {:source-table card-virtual-table-id:aggregation [["count"]]}}]diff --git a/test/metabase/xrays/automagic_dashboards/core_test.clj b/test/metabase/xrays/automagic_dashboards/core_test.cljindex 31ac46d4d96..356c8dfc29d 100644--- a/test/metabase/xrays/automagic_dashboards/core_test.clj+++ b/test/metabase/xrays/automagic_dashboards/core_test.clj@@ -1598,7 +1598,7 @@first)aggregate (get-in dashcard [:card :dataset_query :query :aggregation])](testing "Fields requiring a join should have :source-field populated in the aggregate."- (is (= [["distinct" [:field (mt/id :products :id)- ;; This should be present vs. nil (value before issue)- {:source-field (mt/id :reviews :product_id)}]]]+ (is (= [[:distinct [:field (mt/id :products :id)+ ;; This should be present vs. nil (value before issue)+ {:source-field (mt/id :reviews :product_id)}]]]aggregate)))))))diff --git a/test/metabase/xrays/automagic_dashboards/names_test.clj b/test/metabase/xrays/automagic_dashboards/names_test.cljindex f2e38327b0f..ab44c6685a2 100644--- a/test/metabase/xrays/automagic_dashboards/names_test.clj+++ b/test/metabase/xrays/automagic_dashboards/names_test.clj@@ -36,7 +36,7 @@:week-of-year (u.date/extract dt :week-of-year)}](testing (format "unit = %s" unit)(is (= (str expected)- (str (names/humanize-datetime t-str unit))))))))+ (str (#'names/humanize-datetime t-str unit))))))))(testing "Extracted unit handling"(is (= :sunday (#'u.date/start-of-week)) "adjust the test, it assumes Sunday as first day of the week")(let [t 3] ; t = 2 or t = 4 would also work@@ -49,11 +49,11 @@:week-of-year t}](testing (format "unit = %s" unit)(is (= (str expected)- (str (names/humanize-datetime t unit)))))))))+ (str (#'names/humanize-datetime t unit)))))))))(deftest ^:parallel pluralize-test(are [expected n] (= (str expected)- (str (names/pluralize n)))+ (str (#'names/pluralize n)))(tru "{0}st" 1) 1(tru "{0}nd" 22) 22(tru "{0}rd" 303) 303@@ -65,7 +65,7 @@(doseq [unit (disj (set (concat u.date/extract-units u.date/truncate-units)):iso-day-of-year :second-of-minute :millisecond)](testing unit- (is (some? (names/humanize-datetime "1990-09-09T12:30:00" unit)))))))+ (is (some? (#'names/humanize-datetime "1990-09-09T12:30:00" unit)))))));;; ------------------- Cell titles -------------------(deftest ^:parallel cell-title-test@@ -181,7 +181,7 @@(mt/dataset test-data(is (= "TestColumn is 2 and Created At is in February 2024"(#'names/humanize-filter-value- nil+ {:database (mt/id)}["and"["=" ["expression" "TestColumn" {:base-type "type/Integer"}] 2]["=" ["field" (mt/id :orders :created_at)diff --git a/test_resources/serialization_baseline/collections/cards/aqpA3mIKUnfzYUlwjuGwT_source_question.yaml b/test_resources/serialization_baseline/collections/cards/aqpA3mIKUnfzYUlwjuGwT_source_question.yamlindex 33217f79389..a88eb7eea51 100644--- a/test_resources/serialization_baseline/collections/cards/aqpA3mIKUnfzYUlwjuGwT_source_question.yaml+++ b/test_resources/serialization_baseline/collections/cards/aqpA3mIKUnfzYUlwjuGwT_source_question.yaml@@ -20,7 +20,7 @@ parameter_mappings: []dataset_query:database: My Databasenative:- native: select 1+ query: select 1type: nativeresult_metadata: nullvisualization_settings: