The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Require typed, closed argument schemas in mu/defn (#82447)
Alexander Polyankine79f40e3d365
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 84472144430..8f0af4013b6 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -116,7 +116,10 @@utilwarehouse-schema-overlaywarehouses}- :model-exports #{:model/Action}+ :model-exports #{:model/Action+ :model/HTTPAction+ :model/ImplicitAction+ :model/QueryAction}:model-imports #{:model/Card:model/Collection:model/Dashboard@@ -262,6 +265,7 @@eventslib.schemamodels+ requestsettingsutilutil.config}@@ -437,8 +441,15 @@:api #{metabase.batch-processing.coremetabase.batch-processing.init}:uses #{app-db+ parameters+ query-processor.cache-backendsettings- util}}+ util}+ :model-imports #{:model/Card+ :model/Collection+ :model/Dashboard+ :model/Document+ :model/Table}}bookmarks{:team "UX West"@@ -505,6 +516,7 @@metabase.channel.initmetabase.channel.models.channelmetabase.channel.render.core+ metabase.channel.schemametabase.channel.settingsmetabase.channel.slackmetabase.channel.template.core@@ -672,11 +684,11 @@content-verification{:team "UX West"- :api #{metabase.content-verification.core- metabase.content-verification.init}+ :api #{metabase.content-verification.core metabase.content-verification.init metabase.content-verification.schema}:uses #{app-dblib.schemamodels+ usersutil}:model-exports #{:model/ModerationReview}:model-imports #{:model/Card :model/Dashboard :model/User}}@@ -734,14 +746,17 @@metabase.dashboards.schemametabase.dashboards.settingsmetabase.dashboards.write}- :uses #{analytics+ :uses #{actions+ analyticsapiapp-dbaudit-appcollections+ content-verificationembeddingeventslib+ lib.belib.schemamodelsparameters@@ -786,6 +801,7 @@dashboardsembeddingevents+ liblib.belib.schemamodels@@ -1004,7 +1020,7 @@explorations{:team "UX West"- :api #{metabase.explorations.core metabase.explorations.init}+ :api #{metabase.explorations.core metabase.explorations.init metabase.explorations.schema}:uses #{analyticsanalytics.interfaceapi@@ -1126,8 +1142,8 @@interestingness{:team "UX West"- :api #{metabase.interestingness.core}- :uses #{util}}+ :api #{metabase.interestingness.chart.types metabase.interestingness.core}+ :uses #{lib.be util}}lib{:team "Querying Platform"@@ -1148,6 +1164,7 @@metabase.lib.normalizemetabase.lib.optionsmetabase.lib.pivot+ metabase.lib.refmetabase.lib.types.isametabase.lib.utilmetabase.lib.walk}@@ -1394,14 +1411,17 @@eventsliblib.be+ lib.metriclib.schemametricsmodelspermissionsremote-syncsearch+ usersutil- warehouse-schema-overlay}+ warehouse-schema-overlay+ warehouses.schema}:model-exports #{:model/Measure}:model-imports #{:model/Collection :model/Table :model/User}}@@ -1479,7 +1499,8 @@:model/Metabot:model/MetabotConversation:model/MetabotFeedback- :model/MetabotMessage}+ :model/MetabotMessage+ :model/MetabotPrompt}:model-imports #{:model/Card:model/Collection:model/Dashboard@@ -1672,20 +1693,53 @@lib.legacy-mbqllib.metadatalib.schema+ parametersutilutil.classloaderwarehouse-schema-overlay}- :model-imports #{:model/Card+ :model-imports #{:model/Action+ :model/Card+ :model/Channel+ :model/ChannelTemplate:model/Collection+ :model/CustomVizPlugin:model/Dashboard+ :model/DashboardCard+ :model/DashboardCardSeries:model/DashboardTab:model/Database+ :model/Dimension:model/Document+ :model/EmbeddingTheme+ :model/Exploration:model/Field+ :model/FieldUserSettings+ :model/FieldValues+ :model/Glossary+ :model/HTTPAction+ :model/ImplicitAction:model/Measure+ :model/Metabot+ :model/MetabotPrompt:model/NativeQuerySnippet+ :model/Notification+ :model/NotificationCard+ :model/NotificationHandler+ :model/NotificationRecipient+ :model/NotificationSubscription+ :model/OsiAiContext+ :model/PythonLibrary+ :model/QueryAction:model/Segment:model/Table+ :model/TableIndex+ :model/Timeline+ :model/TimelineEvent+ :model/Transform+ :model/TransformJob+ :model/TransformJobTransformTag+ :model/TransformTag+ :model/TransformTransformTag:model/User}}mq@@ -1701,8 +1755,7 @@native-query-snippets{:team "Graphy"- :api #{metabase.native-query-snippets.api- metabase.native-query-snippets.core}+ :api #{metabase.native-query-snippets.api metabase.native-query-snippets.core metabase.native-query-snippets.schema}:uses #{apicollectionsevents@@ -1712,6 +1765,7 @@permissionspremium-featuresremote-sync+ usersutil}:model-exports #{:model/NativeQuerySnippet}:model-imports #{:model/Collection :model/User}}@@ -1754,8 +1808,10 @@utilutil.config}:model-exports #{:model/Notification+ :model/NotificationCard:model/NotificationHandler- :model/NotificationRecipient}+ :model/NotificationRecipient+ :model/NotificationSubscription}:model-imports #{:model/Card:model/Channel:model/ChannelTemplate@@ -1796,6 +1852,7 @@metabase.parameters.shared}:uses #{apiapp-db+ dashboardsliblib.belib.metadata@@ -1835,6 +1892,8 @@utilutil.configwarehouse-schema-overlay+ warehouses+ warehouses.schemaenterprise/permissions}:model-exports #{:model/ApplicationPermissionsRevision:model/CollectionPermissionGraphRevision@@ -2266,6 +2325,7 @@lib.schemapermissionspremium-features+ serversessionsettingsusers@@ -2282,13 +2342,17 @@collectionsdashboardsevents+ explorationsliblib.belib.schema+ measuresmodelsparametersqueriesquery-permissions+ segments+ transformsutilutil.config}:model-exports #{:model/Revision}@@ -2406,8 +2470,10 @@permissionsremote-syncsearch+ usersutilwarehouse-schema-overlay+ warehouses.schemaxrays}:model-exports #{:model/Segment}:model-imports #{:model/Collection :model/Table :model/User}}@@ -2455,6 +2521,7 @@:api #{metabase.session.apimetabase.session.coremetabase.session.init+ metabase.session.schemametabase.session.settings} ; TODO -- shouldn't be exposed as an API namespace:uses #{apiapp-db@@ -2606,7 +2673,8 @@staleness{:team "UX West"- :api #{metabase.staleness.core}}+ :api #{metabase.staleness.core}+ :uses #{util}}startup{:team "DevEx"@@ -2767,7 +2835,7 @@lib.schemamodelsutil}- :model-exports #{:model/Timeline}+ :model-exports #{:model/Timeline :model/TimelineEvent}:model-imports #{:model/Collection :model/User}}tracing@@ -2826,7 +2894,8 @@:model/TransformJobRun:model/TransformJobTransformTag:model/TransformRun- :model/TransformTag}+ :model/TransformTag+ :model/TransformTransformTag}:model-imports #{:model/Collection:model/Database:model/Field@@ -2868,11 +2937,12 @@transforms.indexes{:team "Gadget":ns-prefix "metabase.indexes"- :api #{metabase.indexes.models.table-index metabase.indexes.reconcile}+ :api #{metabase.indexes.models.table-index metabase.indexes.reconcile metabase.indexes.schema}:uses #{driverlib.schemamodels- util}+ util+ warehouses}:model-exports #{:model/TableIndex}}transforms.indexes.rest@@ -3651,14 +3721,18 @@liblib.belib.metadata+ lib.metriclib.schema+ measuresmodels+ native-query-snippetspermissionspremium-featuresqueriesquery-processorrequestrevisions+ segmentssettingssql-toolstask@@ -3666,7 +3740,8 @@transforms.baseutilutil.graph- warehouse-schema-overlay}+ warehouse-schema-overlay+ enterprise/permissions.sandbox}:model-imports #{:model/Card:model/Collection:model/Dashboard@@ -3696,6 +3771,7 @@measuresmodel-persistencemodels+ parametersqueriessegmentstask@@ -3963,7 +4039,7 @@enterprise/permissions.sandbox{:team "UX West":ns-prefix "metabase-enterprise.sandbox"- :api #{metabase-enterprise.sandbox.api.routes metabase-enterprise.sandbox.api.util}+ :api #{metabase-enterprise.sandbox.api.routes metabase-enterprise.sandbox.api.util metabase-enterprise.sandbox.schema}:uses #{apiapp-dbaudit-appdiff --git a/dev/src/dev/add_load.clj b/dev/src/dev/add_load.cljindex a7ef674bedf..49618079cb9 100644--- a/dev/src/dev/add_load.clj+++ b/dev/src/dev/add_load.clj@@ -1,16 +1,30 @@(ns dev.add-load(:require [clojure.walk :as walk][dev.with-perm :as perm]- [metabase.util.malli :as mu]))+ [metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr]))(def ^:private logical-kw"A keyword that starts with a question mark, used to track values created by from-script."[:and :keyword [:fn #(= "?" (namespace %))]])+(def ^:private Attrs+ "The attributes of an entity to insert, as `mt/with-temp` takes them for its model."+ [:map {:closed false, ::mr/deliberately-open true, :description "model attributes"}])++(def ^:private Bindings+ "A map of logical keywords to the columns of an inserted entity they bind."+ [:map {:closed false, ::mr/deliberately-open true, :description "logical keyword bindings"}])++(def ^:private Inserted+ "An inserted entity of any model."+ [:map {:closed false, ::mr/deliberately-open true, :description "inserted entity"}])+(mu/defn- extract-bindings"Extracts the bindings from the script and inserts the values into the ids map. This is the data oriented equivalentof let."- [bindings :- [:or :map logical-kw] inserted]+ [bindings :- [:or Bindings logical-kw]+ inserted :- Inserted](cond (map? bindings)(update-vals bindings #(get inserted %))(and (keyword? bindings) (= "?" (namespace bindings)))@@ -18,14 +32,15 @@(mu/defn- fill-attrs"Data oriented value resolution used to add attributes to a model from the [[attrs]] map."- [ids :- :map attrs :- :map]+ [ids :- Bindings+ attrs :- Attrs](walk/postwalk(fn [x] (if-let [value (get ids x)] value x))attrs))-(def ^:private BindingForm [:or [:= :?] logical-kw :map])+(def ^:private BindingForm [:or [:= :?] logical-kw Bindings])-(def ^:private NormalizedTuple [:tuple :keyword BindingForm :map])+(def ^:private NormalizedTuple [:tuple :keyword BindingForm Attrs])(def ^:private Tuple [:or[:tuple :keyword]diff --git a/dev/src/dev/debug_test_data.clj b/dev/src/dev/debug_test_data.cljindex fac41b10747..690d0752961 100644--- a/dev/src/dev/debug_test_data.clj+++ b/dev/src/dev/debug_test_data.clj@@ -22,7 +22,7 @@(mu/defn ->dbdef :- [:map [:database-name :string]]"Coerce something to a Database Definition. Can resolve string symbols from the current namespace orfrom [[metabase.test.data.dataset-definitions]]."- [dbdeffable]+ [dbdeffable :- [:or :symbol tx/ValidDatabaseDefinition]](if (symbol? dbdeffable)(->dbdef (data.impl/resolve-dataset-definition (symbol (ns-name *ns*)) dbdeffable))(tx/get-dataset-definition dbdeffable)))@@ -30,7 +30,8 @@(mu/defn ->tabledef :- [:map [:table-name :string]]"Coerce something to a Table Definition. You can pass in a string table name to find the matching table from aDatabase Definition."- [dbdeffable tabledeffable]+ [dbdeffable :- [:or :symbol tx/ValidDatabaseDefinition]+ tabledeffable :- [:or :string tx/ValidTableDefinition]](cond(and (map? tabledeffable)(:table-name tabledeffable))diff --git a/dev/src/dev/deps_graph.clj b/dev/src/dev/deps_graph.cljindex fb45469a12f..3ef0ccd78f4 100644--- a/dev/src/dev/deps_graph.clj+++ b/dev/src/dev/deps_graph.clj@@ -22,6 +22,12 @@;; mechanism for bounded caching of those parsed files.(def ^:dynamic *parsed-file-cache* nil)+(def ^:private File+ "A path string or `java.io.File`, as accepted by `rewrite-clj.parser/parse-file-all` and friends."+ [:or+ string?+ [:fn {:error/message "Instance of a java.io.File"} #(instance? java.io.File %)]])+(defn- parse-file-all"Calls `rewrite-.clj.parser/parse-file-all`, but first checks in `*parsed-file-cache*` if it is bound."[file]@@ -121,7 +127,7 @@(mu/defn- find-dynamically-loaded-namespaces :- [:set simple-symbol?]"Find the set of namespace symbols for namespaces loaded by `require` and friends in a `file`."- [file]+ [file :- File](try(find-required-namespaces file)(catch Throwable e@@ -142,7 +148,7 @@(mu/defn find-defenterprises"using rewrite-clj, find and return all the namespaces 'required' by defenterprise forms in a file."- [file]+ [file :- :string];; We want to know what namespace defendpoint 'requires': so do not need to parse anything in the enterprise dir.(if (str/includes? file "/metabase_enterprise/")[]@@ -160,7 +166,7 @@(mu/defn find-defenterprise-schemas"using rewrite-clj, find and return all the namespaces 'required' by defenterprise-schema forms in a file."- [file]+ [file :- :string];; We want to know what namespace defendpoint 'requires': so do not need to parse anything in the enterprise dir.(if (str/includes? file "/metabase_enterprise/")[]@@ -204,19 +210,37 @@don't include them in our deps tree."'{metabase.config.core #{metabase-enterprise.core.dummy-namespace metabase.test.dummy-namespace}})-(mu/defn- file-dependencies :- [:map- [:namespace simple-symbol?]- [:filename string?] ; filename is relative to [[project-root]]- [:module symbol?]- [:deps [:sequential- [:map- [:namespace simple-symbol?]- [:module symbol?]- [:dynamic {:optional true} :keyword]]]]]- [prefix->module :- map?- file :- [:or- string?- [:fn {:error/message "Instance of a java.io.File"} #(instance? java.io.File %)]]]+(def ^:private ModuleConfig+ [:map {:closed true}+ [:team {:optional true} :string]+ [:ns-prefix {:optional true} :string]+ [:api {:optional true} [:or [:= :any] [:set :symbol]]]+ [:uses {:optional true} [:or [:= :any] [:set :symbol]]]+ [:friends {:optional true} [:set :symbol]]+ [:model-exports {:optional true} [:or [:= :any] [:set :keyword]]]+ [:model-imports {:optional true} [:or [:= :bypass] [:set :keyword]]]+ [:module-exports {:optional true} [:set :symbol]]])++(def ^:private ModulesConfig+ [:map-of :symbol ModuleConfig])++(def ^:private PrefixToModule+ [:map-of :string :symbol])++(def ^:private FileDependencies+ [:map {:closed true}+ [:namespace {:optional true} [:maybe simple-symbol?]]+ [:filename string?] ; filename is relative to [[project-root]]+ [:module symbol?]+ [:deps [:sequential+ [:map {:closed true}+ [:namespace simple-symbol?]+ [:module symbol?]+ [:dynamic {:optional true} :keyword]]]]])++(mu/defn- file-dependencies :- FileDependencies+ [prefix->module :- PrefixToModule+ file :- File](try(let [decl (ns.file/read-file-ns-decl file)ns-symb (ns.parse/name-from-ns-decl decl)@@ -612,7 +636,8 @@(mu/defn- module->source-files :- [:set :string]"Return the set of all *source* filenames (relative to the [[project-root]] directory) for a `module`."- [deps module]+ [deps :- [:sequential FileDependencies]+ module :- symbol?](into(sorted-set)(comp (filter #(= (:module %) module))@@ -721,11 +746,11 @@(mu/defn- module->test-files :- [:set :string]"Return the set of test filenames associated with a `module`."- ([modules-config :- map?+ ([modules-config :- ModulesConfigmodule-sym :- :symbol](module->test-files modules-config (modules/build-prefix->module modules-config) module-sym))- ([modules-config :- map?- prefix->module :- map?+ ([modules-config :- ModulesConfig+ prefix->module :- PrefixToModulemodule-sym :- :symbol](let [path-prefix (module->test-path-prefix modules-config module-sym)test-dir (io/file path-prefix)@@ -767,7 +792,7 @@(mu/defn find-model-keywords :- [:set :keyword]"Find all `:model/X` keywords referenced in a source file, ignoring comments."- [file]+ [file :- File](try(let [models (atom #{})](walk-parsed-ignore-comments!@@ -787,7 +812,7 @@(mu/defn find-model-definitions :- [:set :keyword]"Find all models with their `t2/table-name` defined in this file."- [file]+ [file :- File](let [models (atom #{})](walk-parsed-ignore-comments!(fn [node]diff --git a/dev/src/dev/migrate.clj b/dev/src/dev/migrate.cljindex 5b5243bcafe..cdce353675b 100644--- a/dev/src/dev/migrate.clj+++ b/dev/src/dev/migrate.clj@@ -122,8 +122,8 @@#_{:clj-kondo/ignore [:discouraged-var]}(println (format "Rollbacked %d migrations. Latest migration: %s" n (latest-migration)))))- ([k :- [:enum :id :count "id" "count"]- target]+ ([k :- [:enum :id :count "id" "count"]+ target :- [:or :int :string]](let [n (case (keyword k):id (migration-since target):count (maybe-parse-long target))]@@ -195,9 +195,10 @@;; =>{:forward \"DROP INDEX public.idx_user_id_device_id;\",:rollback \"CREATE INDEX idx_user_id_device_id ON public.login_history(session_id, device_id);\"}"- ([id]+ ([id :- :string](migration-sql-by-id id (mdb/db-type)))- ([id db-type :- [:enum :postgres :mysql :mariadb :h2]]+ ([id :- :string+ db-type :- [:enum :postgres :mysql :mariadb :h2]](t2/with-connection [conn](liquibase/with-liquibase [^Liquibase liquibase conn](let [database (liquibase-database db-type)diff --git a/enterprise/backend/src/metabase_enterprise/action_v2/api.clj b/enterprise/backend/src/metabase_enterprise/action_v2/api.cljindex 25a5978beb0..f4f20107cf1 100644--- a/enterprise/backend/src/metabase_enterprise/action_v2/api.clj+++ b/enterprise/backend/src/metabase_enterprise/action_v2/api.clj@@ -2,6 +2,7 @@(:require[clojure.walk :as walk][metabase-enterprise.action-v2.execute-form :as data-editing.execute-form]+ [metabase-enterprise.action-v2.schema :as action-v2.schema][metabase.actions.core :as actions][metabase.actions.types :as types][metabase.api.common :as api]@@ -59,20 +60,7 @@[:fn {:error/message "value must be a scalar, or a sequence of scalars"}#(mr/validate strict-action-value-map %)]])-(mr/def ::action-expression- "The internal representation used by our APIs, after we've parsed the relevant ids and fetched their configuration."- ;; Expected extensions:- ;; - data app actions (with their mappings)- ;; - action expressions (e.g., unsaved data app actions. might not need these with auto save)- ;; - dashboard buttons (unless we deprecate them instead)- [:or- [:map {:closed true}- [:model-action-id ms/PositiveInt]]- [:map {:closed true}- [:action-kw :keyword]- [:mapping [:maybe :map]]]])--(mu/defn- fetch-unified-action :- ::action-expression+(mu/defn- fetch-unified-action :- ::action-v2.schema/action-expression"Resolve various flavors of action-id into plain data, making it easier to dispatch on. Fetch config etc."[scope :- ::types/scope.hydratedraw :- ::api-action-id-or-expression]diff --git a/enterprise/backend/src/metabase_enterprise/action_v2/data_editing.clj b/enterprise/backend/src/metabase_enterprise/action_v2/data_editing.cljindex bc0bd53c0d1..f380073b46a 100644--- a/enterprise/backend/src/metabase_enterprise/action_v2/data_editing.clj+++ b/enterprise/backend/src/metabase_enterprise/action_v2/data_editing.clj@@ -175,7 +175,9 @@(let [table-ids (distinct (map :table-id diffs))table->pk-fields (u/group-by identity select-table-pk-fields concat table-ids)diff->pk-diff (u/for-map [{:keys [table-id before after] :as diff} diffs- :when (or before after)]+ :when (or before after)+ :let [before (some-> before (update-keys u/qualified-name))+ after (some-> after (update-keys u/qualified-name))]][diff {:pk (get-row-pks (table->pk-fields table-id) (or after before)):before before:after after}])]diff --git a/enterprise/backend/src/metabase_enterprise/action_v2/db.clj b/enterprise/backend/src/metabase_enterprise/action_v2/db.cljindex 23d16a01462..0e0ad9ab67d 100644--- a/enterprise/backend/src/metabase_enterprise/action_v2/db.clj+++ b/enterprise/backend/src/metabase_enterprise/action_v2/db.clj@@ -12,7 +12,7 @@(mu/defn table"The Table with `table-id`, or nil."- [table-id :- ::lib.schema.id/table]+ [table-id :- [:maybe ::lib.schema.id/table]](t2/select-one :model/Table :id table-id {:from [(warehouse-schema-overlay/table-query)]}))(mu/defn active-table@@ -146,7 +146,7 @@batch `batch-num` with the opposite undone state."[undo? :- :booleantable-ids :- [:set ::lib.schema.id/table]- row-pks :- [:set :some]+ row-pks :- [:set ::action-v2.schema/undo.row-pk]batch-num :- ms/PositiveInt](t2/exists? :model/Undo:table_id [:in table-ids]diff --git a/enterprise/backend/src/metabase_enterprise/action_v2/execute_form.clj b/enterprise/backend/src/metabase_enterprise/action_v2/execute_form.cljindex 481c16cb355..9354409a65e 100644--- a/enterprise/backend/src/metabase_enterprise/action_v2/execute_form.clj+++ b/enterprise/backend/src/metabase_enterprise/action_v2/execute_form.clj@@ -1,8 +1,12 @@(ns metabase-enterprise.action-v2.execute-form(:require[metabase-enterprise.action-v2.db :as action-v2.db]+ [metabase-enterprise.action-v2.schema :as action-v2.schema][metabase.actions.core :as actions]+ [metabase.actions.types :as actions.types][metabase.api.common :as api]+ [metabase.lib.schema.id :as lib.schema.id]+ [metabase.lib.schema.parameter :as lib.schema.parameter][metabase.util :as u][metabase.util.malli :as mu][metabase.util.malli.registry :as mr]@@ -53,6 +57,15 @@[:title :string][:parameters [:sequential ::describe-param]]])+(mr/def ::partial-input+ "A single input row after `apply-mapping-nested`: either mapped to the target table, or, absent a mapping, the raw+ parameter values it was given."+ [:or+ [:map {:closed true}+ [:table-id {:optional true} ::lib.schema.id/table]+ [:row {:optional true} [:maybe [:map-of :string [:ref ::lib.schema.parameter/parameter.value]]]]]+ [:map-of :string [:ref ::lib.schema.parameter/parameter.value]]])+(defn- field-input-type-ignoring-semantics [create? field field-values](condp #(isa? %2 %1) (:base_type field):type/Boolean :input/boolean@@ -151,7 +164,9 @@(mu/defn describe-form :- ::action-description"Describe parameters of an unified action."- [action-def scope partial-input]+ [action-def :- ::action-v2.schema/action-expression+ scope :- ::actions.types/scope.hydrated+ partial-input :- ::partial-input](cond(:action-id action-def)(throw (ex-info "We do not currently support execution of Model Actions" {:status-code 400}))diff --git a/enterprise/backend/src/metabase_enterprise/action_v2/schema.clj b/enterprise/backend/src/metabase_enterprise/action_v2/schema.cljindex c66e0f882ee..499c30421a9 100644--- a/enterprise/backend/src/metabase_enterprise/action_v2/schema.clj+++ b/enterprise/backend/src/metabase_enterprise/action_v2/schema.clj@@ -6,32 +6,42 @@[metabase.util.malli.schema :as ms]))(mr/def ::undo.row-pk- "The `:row_pk` column of a Undo, decoded."- :map)+ "The `:row_pk` column of a Undo, decoded: a warehouse table's primary-key columns, whose keys and values that table owns."+ [:map {:closed false, ::mr/deliberately-open true}])(mr/def ::undo.raw-before- "The `:raw_before` column of a Undo, decoded."- :map)+ "The `:raw_before` column of a Undo, decoded: a warehouse table row, whose keys and values that table owns."+ [:map {:closed false, ::mr/deliberately-open true}])(mr/def ::undo.raw-after- "The `:raw_after` column of a Undo, decoded."- :map)+ "The `:raw_after` column of a Undo, decoded: a warehouse table row, whose keys and values that table owns."+ [:map {:closed false, ::mr/deliberately-open true}])(mr/def ::undo"A Undo as selected from the app DB: every column of `:data_edit_undo_chain`."+ [:merge+ ::undo.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])++(mr/def ::action-mapping+ "A `:mapping` produced by `default-mapping`: which table the action targets and where the row data plugs in."[:map {:closed true}- [:id ms/PositiveInt]- [:batch_num :int]- [:table_id ::lib.schema.id/table]- [:row_pk ::undo.row-pk]- [:user_id ::lib.schema.id/user]- [:scope [:or :keyword :string]]- [:undoable :boolean]- [:raw_before [:maybe ::undo.raw-before]]- [:raw_after [:maybe ::undo.raw-after]]- [:undone :boolean]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]])+ [:table-id {:optional true} ::lib.schema.id/table]+ [:row :keyword]])++(mr/def ::action-expression+ "The internal representation used by our APIs, after we've parsed the relevant ids and fetched their configuration."+ ;; Expected extensions:+ ;; - data app actions (with their mappings)+ ;; - action expressions (e.g., unsaved data app actions. might not need these with auto save)+ ;; - dashboard buttons (unless we deprecate them instead)+ [:or+ [:map {:closed true}+ [:model-action-id ms/PositiveInt]]+ [:map {:closed true}+ [:action-kw :keyword]+ [:mapping [:maybe ::action-mapping]]]])(mr/def ::undo.update"What an update (or insert) of a Undo accepts: every column of `:data_edit_undo_chain` except `id`, all optional."diff --git a/enterprise/backend/src/metabase_enterprise/advanced_permissions/db.clj b/enterprise/backend/src/metabase_enterprise/advanced_permissions/db.cljindex 61b444a6121..410e640a766 100644--- a/enterprise/backend/src/metabase_enterprise/advanced_permissions/db.clj+++ b/enterprise/backend/src/metabase_enterprise/advanced_permissions/db.clj@@ -11,7 +11,7 @@(mu/defn table"The Table with `table-id`, or nil."- [table-id :- ::lib.schema.id/table]+ [table-id :- [:maybe ::lib.schema.id/table]](t2/select-one :model/Table :id table-id {:from [(warehouse-schema-overlay/table-query)]}))(mu/defn group-manager?diff --git a/enterprise/backend/src/metabase_enterprise/advanced_permissions/models/permissions/application_permissions.clj b/enterprise/backend/src/metabase_enterprise/advanced_permissions/models/permissions/application_permissions.cljindex 700ce7af48d..c443fc98a79 100644--- a/enterprise/backend/src/metabase_enterprise/advanced_permissions/models/permissions/application_permissions.clj+++ b/enterprise/backend/src/metabase_enterprise/advanced_permissions/models/permissions/application_permissions.clj@@ -46,7 +46,7 @@(mu/defn permissions-set->application-perms :- GroupPermissionsGraph"Get a map of all application permissions for a group."- [permission-set]+ [permission-set :- [:maybe [:set :string]]]{:setting (permission-for-type permission-set :setting):monitoring (permission-for-type permission-set :monitoring):subscription (permission-for-type permission-set :subscription)})diff --git a/enterprise/backend/src/metabase_enterprise/audit_app/interface.clj b/enterprise/backend/src/metabase_enterprise/audit_app/interface.cljindex 4a1c21ecabd..84992fb94af 100644--- a/enterprise/backend/src/metabase_enterprise/audit_app/interface.clj+++ b/enterprise/backend/src/metabase_enterprise/audit_app/interface.clj@@ -10,9 +10,12 @@{:min 1}[:tuplems/KeywordOrString- [:map- [:base_type ms/FieldType]- [:display_name ms/NonBlankString]]]])+ [:map {:closed true}+ [:base_type ms/FieldType]+ [:display_name ms/NonBlankString]+ [:remapped_to {:optional true} :keyword]+ [:remapped_from {:optional true} :keyword]+ [:code {:optional true} :boolean]]]])(defmulti internal-query"Define a new internal query type. Conventionally `query-type` should be a namespaced keyword with the namespace indiff --git a/enterprise/backend/src/metabase_enterprise/audit_app/pages/common.clj b/enterprise/backend/src/metabase_enterprise/audit_app/pages/common.cljindex e3b130131b5..4bb44857900 100644--- a/enterprise/backend/src/metabase_enterprise/audit_app/pages/common.clj+++ b/enterprise/backend/src/metabase_enterprise/audit_app/pages/common.clj@@ -16,7 +16,8 @@[metabase.util :as u][metabase.util.honey-sql-2 :as h2x][metabase.util.i18n :refer [tru]]- [metabase.util.malli :as mu]))+ [metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr]))(set! *warn-on-reflection* true)@@ -101,10 +102,37 @@{:driver driver, :honeysql-query honeysql-query}e)))))+(mr/def ::honeysql-expr+ "A Honey SQL 2 expression inside an audit query: a plain expression, a subquery, or a vector form containing these."+ [:or+ ::h2x/expr+ [:ref ::honeysql-query]+ [:sequential [:ref ::honeysql-expr]]])++(mr/def ::honeysql-query+ "A Honey SQL 2 query map as this namespace's audit queries build it: a `:with` CTE's body and a `:union-all`+ branch are each themselves one of these."+ [:map {:closed true}+ [:with {:optional true} [:sequential [:tuple :keyword [:ref ::honeysql-query]]]]+ [:select {:optional true} [:sequential [:ref ::honeysql-expr]]]+ [:select-distinct {:optional true} [:sequential [:ref ::honeysql-expr]]]+ [:from {:optional true} [:sequential [:ref ::honeysql-expr]]]+ [:join {:optional true} [:sequential [:ref ::honeysql-expr]]]+ [:left-join {:optional true} [:sequential [:ref ::honeysql-expr]]]+ [:inner-join {:optional true} [:sequential [:ref ::honeysql-expr]]]+ [:where {:optional true} [:ref ::honeysql-expr]]+ [:group-by {:optional true} [:sequential [:ref ::honeysql-expr]]]+ [:having {:optional true} [:ref ::honeysql-expr]]+ [:partition-by {:optional true} [:sequential [:ref ::honeysql-expr]]]+ [:order-by {:optional true} [:sequential [:ref ::honeysql-expr]]]+ [:limit {:optional true} ::h2x/expr]+ [:offset {:optional true} ::h2x/expr]+ [:union-all {:optional true} [:sequential [:ref ::honeysql-query]]]])+(mu/defn- reduce-results* :- :some- [honeysql-query :- :map+ [honeysql-query :- ::honeysql-queryrff :- ::qp.schema/rff- init]+ init :- ::qp.schema/accumulator](let [driver (mdb/db-type)[sql & params] (compile-honeysql driver honeysql-query)];; MySQL driver normalizies timestamps. Setting `*results-timezone-id-override*` is a shortcut@@ -149,7 +177,7 @@`metabase-enterprise.audit-app.query-processor.middleware.handle-audit-queries.internal-queries`)"[honeysql-query](let [rff (fn rff [{:keys [cols]}]- (let [col-names (mapv (comp keyword :name) cols)]+ (let [col-names (mapv :name cols)]((map (partial zipmap col-names)) conj)))](reduce-results* honeysql-query rff [])))diff --git a/enterprise/backend/src/metabase_enterprise/audit_app/query_processor/middleware/handle_audit_queries.clj b/enterprise/backend/src/metabase_enterprise/audit_app/query_processor/middleware/handle_audit_queries.cljindex fa25aa57f66..674e7c20577 100644--- a/enterprise/backend/src/metabase_enterprise/audit_app/query_processor/middleware/handle_audit_queries.clj+++ b/enterprise/backend/src/metabase_enterprise/audit_app/query_processor/middleware/handle_audit_queries.clj@@ -57,7 +57,7 @@"Primarily for dev and debugging purposes. We can probably take this out when shipping the finished product."[results metadata](let [results-keys (set (keys (first results)))- metadata-keys (set (map (comp keyword first) metadata))]+ metadata-keys (set (map (comp name first) metadata))](when (and (seq results-keys)(not= results-keys metadata-keys))(let [[only-in-results only-in-metadata] (data/diff results-keys metadata-keys)]@@ -71,22 +71,25 @@(defn- metadata->cols [metadata](for [[k v] metadata]- (assoc v :name (name k))))+ (cond-> (assoc v :name (name k))+ (:remapped_to v) (update :remapped_to name)+ (:remapped_from v) (update :remapped_from name))))-(mu/defn- format-results [{:keys [results metadata]} :- [:map- [:results [:sequential :map]]+(mu/defn- format-results [{:keys [results metadata]} :- [:map {:closed true}+ [:results [:sequential [:map-of :string ms/FieldValue]]][:metadata audit.i/ResultsMetadata]]](check-results-and-metadata-keys-match results metadata){:cols (metadata->cols metadata):rows (for [row results](for [[k] metadata]- (get row (keyword k))))})+ (get row (name k))))})(def InternalQuery"Schema for a valid `internal` type query."(into [:multi {:dispatch :fn}](cond-> [["metabase-enterprise.audit-app.pages.queries/bad-table"- [:map+ [:map {:closed true}+ [:fn [:= "metabase-enterprise.audit-app.pages.queries/bad-table"]][:type [:enum :internal "internal"]][:args {:optional true}[:maybe [:or@@ -98,15 +101,20 @@"last_run_at" "total_runs" "num_dashboards" "user_id" "user_name""updated_at"]][:maybe [:enum "asc" "desc"]]]]]]- [:limit {:optional true} [:maybe ms/IntGreaterThanOrEqualToZero]]- [:offset {:optional true} [:maybe ms/IntGreaterThanOrEqualToZero]]]]]+ [:limit {:optional true} [:maybe ms/IntGreaterThanOrEqualToZero]]+ [:offset {:optional true} [:maybe ms/IntGreaterThanOrEqualToZero]]+ [:info {:optional true} [:maybe :metabase.lib.schema.info/info]]+ [:middleware {:optional true} [:maybe :metabase.lib.schema.middleware-options/middleware-options]]]]]config/is-test?(conj [::mc/default- [:map- [:type [:enum :internal "internal"]]- [:args {:optional true} [:maybe [:sequential :any]]]- [:limit {:optional true} [:maybe ms/IntGreaterThanOrEqualToZero]]- [:offset {:optional true} [:maybe ms/IntGreaterThanOrEqualToZero]]]]))))+ [:map {:closed true}+ [:fn :string]+ [:type [:enum :internal "internal"]]+ [:args {:optional true} [:maybe [:sequential [:maybe [:or :string number? :boolean]]]]]+ [:limit {:optional true} [:maybe ms/IntGreaterThanOrEqualToZero]]+ [:offset {:optional true} [:maybe ms/IntGreaterThanOrEqualToZero]]+ [:info {:optional true} [:maybe :metabase.lib.schema.info/info]]+ [:middleware {:optional true} [:maybe :metabase.lib.schema.middleware-options/middleware-options]]]]))))(defn- validate-internal-query[query]@@ -131,7 +139,8 @@(qp.pipeline/*reduce* rff* {:cols cols} reducible-rows)))(defn- reduce-legacy-results [rff results]- (let [{:keys [cols rows]} (format-results results)]+ (let [results (update results :results (partial map #(update-keys % name)))+ {:keys [cols rows]} (format-results results)](assert (some? cols))(assert (some? rows))(qp.pipeline/*reduce* rff {:cols cols} rows)))@@ -142,7 +151,7 @@reduce-legacy-results) rff results))(mu/defn- process-internal-query- [{qualified-fn-str :fn, args :args, :as query}+ [{qualified-fn-str :fn, args :args, :as query} :- InternalQueryrff :- ::qp.schema/rff](validate-internal-query query);; Make sure current user is a superuser or has monitoring permissionsdiff --git a/enterprise/backend/src/metabase_enterprise/custom_viz_plugin/models/custom_viz_plugin.clj b/enterprise/backend/src/metabase_enterprise/custom_viz_plugin/models/custom_viz_plugin.cljindex 0000f93ccd2..fb2196c1581 100644--- a/enterprise/backend/src/metabase_enterprise/custom_viz_plugin/models/custom_viz_plugin.clj+++ b/enterprise/backend/src/metabase_enterprise/custom_viz_plugin/models/custom_viz_plugin.clj@@ -2,6 +2,7 @@(:require[buddy.core.codecs :as codecs][metabase-enterprise.custom-viz-plugin.db :as custom-viz-plugin.db]+ [metabase-enterprise.custom-viz-plugin.schema][metabase.api.common :as api][metabase.models.interface :as mi][metabase.models.serialization :as serdes]diff --git a/enterprise/backend/src/metabase_enterprise/custom_viz_plugin/schema.clj b/enterprise/backend/src/metabase_enterprise/custom_viz_plugin/schema.cljindex c41139b9e7d..ebd58c5255d 100644--- a/enterprise/backend/src/metabase_enterprise/custom_viz_plugin/schema.clj+++ b/enterprise/backend/src/metabase_enterprise/custom_viz_plugin/schema.clj@@ -6,25 +6,18 @@(mr/def ::custom-viz-plugin.manifest"The `:manifest` column of a CustomVizPlugin, decoded."- :map)+ [:map {:closed true}+ [:name {:optional true} [:maybe :string]]+ [:icon {:optional true} [:maybe :string]]+ [:metabase {:optional true} [:maybe [:map {:closed true} [:version {:optional true} [:maybe :string]]]]]+ [:sdk {:optional true} [:maybe [:map {:closed true} [:version {:optional true} [:maybe :string]]]]]])(mr/def ::custom-viz-plugin"A CustomVizPlugin as selected from the app DB: every column of `:custom_viz_plugin`."- [:map {:closed true}- [:id ms/PositiveInt]- [:identifier :string]- [:display_name :string]- [:status [:or :keyword :string]]- [:error_message [:maybe :string]]- [:bundle [:maybe [:or bytes? :string]]]- [:bundle_hash [:maybe :string]]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]- [:enabled :boolean]- [:icon [:maybe :string]]- [:manifest [:maybe ::custom-viz-plugin.manifest]]- [:metabase_version [:maybe :string]]- [:dev_bundle_url [:maybe :string]]])+ [:merge+ ::custom-viz-plugin.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::custom-viz-plugin.update"What an update (or insert) of a CustomVizPlugin accepts: every column of `:custom_viz_plugin` except `id`, all optional."diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/schema.clj b/enterprise/backend/src/metabase_enterprise/data_apps/schema.cljindex e93e7341708..eb3f107f2b6 100644--- a/enterprise/backend/src/metabase_enterprise/data_apps/schema.clj+++ b/enterprise/backend/src/metabase_enterprise/data_apps/schema.clj@@ -6,25 +6,10 @@(mr/def ::data-app"A DataApp as selected from the app DB: every column of `:data_app`."- [:map {:closed true}- [:id ms/PositiveInt]- [:name :string]- [:display_name :string]- [:allowed_hosts [:maybe [:sequential :string]]]- [:bundle_path :string]- [:bundle [:maybe [:or bytes? :string]]]- [:bundle_hash [:maybe :string]]- [:last_synced_sha [:maybe :string]]- [:last_synced_at [:maybe ms/TemporalInstant]]- [:sync_error [:maybe :string]]- [:enabled :boolean]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]- [:description [:maybe :string]]- [:resource_collection_id [:maybe ms/PositiveInt]]- [:permission_group_id [:maybe ms/PositiveInt]]- [:table_ids [:sequential ms/PositiveInt]]- [:draft :boolean]])+ [:merge+ ::data-app.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::data-app.update"What an update (or insert) of a DataApp accepts: every column of `:data_app` except `id`, all optional."diff --git a/enterprise/backend/src/metabase_enterprise/data_complexity_score/models/data_complexity_score.clj b/enterprise/backend/src/metabase_enterprise/data_complexity_score/models/data_complexity_score.cljindex 628f4124d93..620bc5da350 100644--- a/enterprise/backend/src/metabase_enterprise/data_complexity_score/models/data_complexity_score.clj+++ b/enterprise/backend/src/metabase_enterprise/data_complexity_score/models/data_complexity_score.clj@@ -1,7 +1,10 @@(ns metabase-enterprise.data-complexity-score.models.data-complexity-score"Persistence for cached Data Complexity Score snapshots."(:require+ [malli.core :as mc]+ [malli.transform :as mtx][metabase-enterprise.data-complexity-score.db :as data-complexity-score.db]+ [metabase-enterprise.data-complexity-score.schema :as data-complexity-score.schema][metabase.models.interface :as mi][methodical.core :as methodical][toucan2.core :as t2]))@@ -11,8 +14,16 @@(doto :model/DataComplexityScore(derive :metabase/model))+(defn- score-data-out+ "Re-keywordize the `:keyword`-typed leaves (e.g. `:meta.text-variant`) that JSON storage round-trips back as+ strings."+ [score-data]+ (when-let [decoded ((:out mi/transform-json) score-data)]+ (mc/decode ::data-complexity-score.schema/data-complexity-score.score-data decoded+ (mtx/transformer mtx/json-transformer))))+(t2/deftransforms :model/DataComplexityScore- {:score_data mi/transform-json})+ {:score_data {:in (:in mi/transform-json), :out score-data-out}})(defn- score-with-calculated-at[{:keys [score_data created_at]}]diff --git a/enterprise/backend/src/metabase_enterprise/data_complexity_score/schema.clj b/enterprise/backend/src/metabase_enterprise/data_complexity_score/schema.cljindex f0e6b0df627..ca7373c45f8 100644--- a/enterprise/backend/src/metabase_enterprise/data_complexity_score/schema.clj+++ b/enterprise/backend/src/metabase_enterprise/data_complexity_score/schema.clj@@ -4,18 +4,63 @@[metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms]))+(mr/def ::data-complexity-score.leaf+ "A `component-score` leaf: a computed `:measurement`/`:score` pair, or an `:error` when computing it failed."+ [:or+ [:map {:closed true}+ [:measurement :double]+ [:score number?]]+ [:map {:closed true}+ [:error :string]]])++(mr/def ::data-complexity-score.catalog+ "One catalog's `score-catalog` result: a `:size`/`:ambiguity` rollup of `::data-complexity-score.leaf`s. A cached+ snapshot re-published without recomputing may carry an empty `:components`."+ [:map {:closed true}+ [:score [:maybe number?]]+ [:components+ [:map {:closed true}+ [:size {:optional true} [:map {:closed true}+ [:score [:maybe number?]]+ [:components [:map {:closed true}+ [:entity-count ::data-complexity-score.leaf]+ [:field-count ::data-complexity-score.leaf]]]]]+ [:ambiguity {:optional true} [:map {:closed true}+ [:score [:maybe number?]]+ [:components [:map {:closed true}+ [:name-collisions ::data-complexity-score.leaf]+ [:synonym-pairs ::data-complexity-score.leaf]+ [:repeated-measures ::data-complexity-score.leaf]]]]]]]])+(mr/def ::data-complexity-score.score-data"The `:score_data` column of a DataComplexityScore, decoded."- :map)+ [:map {:closed true}+ [:library ::data-complexity-score.catalog]+ [:universe ::data-complexity-score.catalog]+ [:metabot ::data-complexity-score.catalog]+ [:meta [:map {:closed true}+ [:formula-version :int]+ [:format-version :int]+ [:synonym-threshold number?]+ [:weights [:map {:closed true}+ [:entity :int]+ [:name-collision :int]+ [:synonym-pair :int]+ [:field :int]+ [:repeated-measure :int]]]+ [:embedding-model {:optional true} [:maybe [:map {:closed true}+ [:provider [:maybe [:or :string :keyword]]]+ [:model-name [:maybe :string]]+ [:model-dimensions [:maybe :int]]]]]+ [:text-variant {:optional true} [:maybe :keyword]]+ [:metabot-source {:optional true} [:maybe :keyword]]]]])(mr/def ::data-complexity-score"A DataComplexityScore as selected from the app DB: every column of `:data_complexity_score`."- [:map {:closed true}- [:id ms/PositiveInt]- [:fingerprint :string]- [:score_data ::data-complexity-score.score-data]- [:created_at ms/TemporalInstant]- [:source [:or :keyword :string]]])+ [:merge+ ::data-complexity-score.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::data-complexity-score.update"What an update (or insert) of a DataComplexityScore accepts: every column of `:data_complexity_score` except `id`, all optional."diff --git a/enterprise/backend/src/metabase_enterprise/data_studio/permissions/published_tables.clj b/enterprise/backend/src/metabase_enterprise/data_studio/permissions/published_tables.cljindex a08a3329b5c..5c5977505d6 100644--- a/enterprise/backend/src/metabase_enterprise/data_studio/permissions/published_tables.clj+++ b/enterprise/backend/src/metabase_enterprise/data_studio/permissions/published_tables.clj@@ -36,7 +36,7 @@:feature :library[table](when (:is_published table)- (mi/current-user-has-full-permissions? (perms/perms-objects-set-for-parent-collection table :read))))+ (mi/current-user-has-full-permissions? (perms/perms-objects-set-for-parent-collection (:collection_id table) :read))))(defenterprise published-table-visible-clause"Returns a HoneySQL clause matching published tables that are readable via collection permissions."diff --git a/enterprise/backend/src/metabase_enterprise/database_routing/schema.clj b/enterprise/backend/src/metabase_enterprise/database_routing/schema.cljindex e624b05237b..6ce6652986f 100644--- a/enterprise/backend/src/metabase_enterprise/database_routing/schema.clj+++ b/enterprise/backend/src/metabase_enterprise/database_routing/schema.clj@@ -7,10 +7,10 @@(mr/def ::database-router"A DatabaseRouter as selected from the app DB: every column of `:db_router`."- [:map {:closed true}- [:id ms/PositiveInt]- [:database_id ::lib.schema.id/database]- [:user_attribute :string]])+ [:merge+ ::database-router.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::database-router.update"What an update (or insert) of a DatabaseRouter accepts: every column of `:db_router` except `id`, all optional."diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/analysis.clj b/enterprise/backend/src/metabase_enterprise/dependencies/analysis.cljindex 6e32099704c..e8bb414e9fe 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/analysis.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/analysis.clj@@ -7,6 +7,7 @@[metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.metadata :as lib.schema.metadata][metabase.lib.schema.validate :as lib.schema.validate]+ [metabase.transforms-base.util :as transforms-base.u][metabase.util.malli :as mu]))(mu/defn returned-columns@@ -45,27 +46,27 @@driver (:engine (lib.metadata/database query))](check-query driver query)))-(mu/defmethod -check-entity :transform :- [:set [:ref ::lib.schema.validate/error-with-source]]+(mu/defmethod -check-entity :transform :- [:maybe [:set [:ref ::lib.schema.validate/error-with-source]]][metadata-provider :- ::lib.schema.metadata/metadata-provider_entity-typetransform-id :- ::lib.schema.id/transform]- (let [{{query :query} :source- :as _transform} (lib.metadata/transform metadata-provider transform-id)- driver (:engine (lib.metadata/database metadata-provider))- query (lib/query metadata-provider query)- output-fields (returned-columns driver query)- ;; Group by the user-visible column name (the alias when present),- ;; not the underlying column's `:name` — otherwise `t1.name AS a` and- ;; `t2.name AS b` look like duplicates because both have `:name "name"`.- output-name #(or (:lib/desired-column-alias %) (:name %))- duplicated-fields (->> output-fields- (group-by output-name)- vals- (keep #(when (> (count %) 1)- (lib/duplicate-column-error (output-name (first %)))))- seq)]- (cond-> (check-query driver query)- duplicated-fields (into duplicated-fields))))+ (let [transform (lib.metadata/transform metadata-provider transform-id)]+ (when (transforms-base.u/query-transform? transform)+ (let [query (lib/query metadata-provider (-> transform :source :query))+ driver (:engine (lib.metadata/database metadata-provider))+ output-fields (returned-columns driver query)+ ;; Group by the user-visible column name (the alias when present),+ ;; not the underlying column's `:name` — otherwise `t1.name AS a` and+ ;; `t2.name AS b` look like duplicates because both have `:name "name"`.+ output-name #(or (:lib/desired-column-alias %) (:name %))+ duplicated-fields (->> output-fields+ (group-by output-name)+ vals+ (keep #(when (> (count %) 1)+ (lib/duplicate-column-error (output-name (first %)))))+ seq)]+ (cond-> (check-query driver query)+ duplicated-fields (into duplicated-fields))))))(mu/defmethod -check-entity :segment :- [:set [:ref ::lib.schema.validate/error-with-source]][metadata-provider :- ::lib.schema.metadata/metadata-providerdiff --git a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj b/enterprise/backend/src/metabase_enterprise/dependencies/api.cljindex 3ccd75bede8..19ca533dfe0 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/api.clj@@ -6,6 +6,7 @@[metabase-enterprise.dependencies.models.analysis-finding-error :as analysis-finding-error][metabase-enterprise.dependencies.models.dependency :as dependency][metabase-enterprise.dependencies.models.dependency-status :as deps.dependency-status]+ [metabase-enterprise.sandbox.schema][metabase.api.common :as api][metabase.api.macros :as api.macros][metabase.api.routes.common :refer [+auth]]@@ -15,8 +16,14 @@[metabase.graph.core :as graph][metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.metadata :as lib.schema.metadata]+ [metabase.lib.schema.validate :as lib.schema.validate]+ [metabase.measures.schema]+ [metabase.native-query-snippets.schema]+ [metabase.queries.schema :as queries.schema][metabase.request.core :as request][metabase.revisions.core :as revisions]+ [metabase.segments.schema]+ [metabase.transforms.schema :as transforms.schema][metabase.util :as u][metabase.util.malli :as mu][metabase.util.malli.registry :as mr]@@ -56,13 +63,23 @@[:enum :table :snippet :transform :dashboard :document :sandbox :segment :question :model :metric :measure]::deps.dependency-types/entity-id])+(mr/def ::dependent-error+ "One entry of `::base-entity`'s `:dependents_errors`: either the slim `{:type :detail}` shape+ [[normalize-finding-error]] produces for an entity's own errors (`node-errors`), or the raw AnalysisFindingError row+ `node-downstream-errors` keeps so callers can tell which downstream entity each error belongs to."+ [:or+ [:map {:closed true}+ [:type ::lib.schema.validate/validate-error-type]+ [:detail {:optional true} [:maybe :string]]]+ ::analysis-finding-error/analysis-finding-error])+(mr/def ::base-entity[:map[:id pos-int?][:type :keyword][:data [:map]][:dependents_count [:maybe [:ref ::usages]]]- [:dependents_errors {:optional true} [:set [:ref ::analysis-finding-error/analysis-finding-error]]]])+ [:dependents_errors {:optional true} [:set [:ref ::dependent-error]]]])(defn- fields-for [entity-key];; these specs should really use something like@@ -147,8 +164,28 @@[:segment [:ref ::segment-entity]][:measure [:ref ::measure-entity]]])+(def ^:private Entity+ "Any dependency-tracked entity `entity-value` accepts, hydrated the way `hydrate-entities` hydrates it."+ [:or+ ::queries.schema/card+ :metabase.dashboards.schema/dashboard+ ::documents.schema/document+ :metabase.warehouse-schema.schema/table+ ::transforms.schema/transform+ :metabase.native-query-snippets.schema/native-query-snippet+ :metabase-enterprise.sandbox.schema/sandbox+ :metabase.segments.schema/segment+ :metabase.measures.schema/measure])+(mu/defn- entity-value :- ::entity- [entity-type {:keys [id] :as entity} usages errors]+ [entity-type :- ::deps.dependency-types/dependency-types+ {:keys [id] :as entity} :- Entity+ usages :- [:maybe [:map-of+ [:tuple ::deps.dependency-types/dependency-types ::deps.dependency-types/entity-id]+ ::usages]]+ errors :- [:maybe [:map-of+ [:tuple ::deps.dependency-types/dependency-types ::deps.dependency-types/entity-id]+ [:set ::dependent-error]]]](cond-> {:id id:type entity-type:data (-> (select-keys entity (entity-keys entity-type))@@ -223,11 +260,17 @@{dependency-type 1})))(apply merge-with +))))))+(defn- normalize-finding-error+ [{:keys [error_type error_detail]}]+ (cond-> {:type error_type}+ error_detail (assoc :detail error_detail)))+(defn- node-downstream-errors"Fetches errors caused by the given source entities (what downstream entities they're breaking).Filters out errors where the analyzed entity is not visible to the current user.Unlike `node-errors` which fetches errors on an entity, this fetches errors that- the entity is causing in other entities that depend on it."+ the entity is causing in other entities that depend on it. Keeps the raw AnalysisFindingError shape (unlike+ `node-errors`'s normalized one) so callers can tell which downstream entity each error belongs to."[nodes-by-type](letfn [(errors-by-source-type-and-id [[source-type ids]](when (seq ids)@@ -244,11 +287,7 @@Filters out errors where the source entity is not visible to the current user.Returns {[entity-type entity-id] #{error-maps...}}, or nil if none."[nodes-by-type]- (letfn [(normalize-finding-error- [{:keys [error_type error_detail]}]- (cond-> {:type error_type}- error_detail (assoc :detail error_detail)))- (errors-by-entity-type-and-id [[type ids]]+ (letfn [(errors-by-entity-type-and-id [[type ids]](when (seq ids)(let [finding-errors (dependencies.db/finding-errors-for-entities-with-visible-sourcestype ids (current-user-visibility nil))]diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj b/enterprise/backend/src/metabase_enterprise/dependencies/calculation.cljindex 937834ed9ac..9c3d7339845 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj@@ -1,16 +1,40 @@(ns metabase-enterprise.dependencies.calculation(:require+ [metabase-enterprise.dependencies.dependency-types :as deps.dependency-types][metabase-enterprise.dependencies.native-validation :as deps.native][metabase-enterprise.dependencies.schema :as deps.schema]+ [metabase-enterprise.sandbox.schema][metabase.documents.prose-mirror :as prose-mirror]+ [metabase.documents.schema :as documents.schema]+ [metabase.lib-be.core :as lib-be]+ [metabase.lib-metric.schema :as lib-metric.schema][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.lib.schema :as lib.schema]+ [metabase.measures.schema]+ [metabase.native-query-snippets.schema]+ [metabase.queries.schema :as queries.schema]+ [metabase.segments.schema][metabase.transforms-base.util :as transforms-base.u]+ [metabase.transforms.schema :as transforms.schema][metabase.util :as u][metabase.util.log :as log][metabase.util.malli :as mu]))+(def ^:private Entity+ "Any dependency-tracked entity `calculate-deps` accepts. A `:dashboard` entity arrives hydrated with `:dashcards`,+ each of those in turn hydrated with `:series`; every other entity is a plain row of its model."+ [:or+ ::queries.schema/card+ :metabase.warehouse-schema.schema/table+ :metabase.native-query-snippets.schema/native-query-snippet+ ::transforms.schema/transform+ :metabase.dashboards.schema/dashboard+ ::documents.schema/document+ :metabase-enterprise.sandbox.schema/sandbox+ :metabase.segments.schema/segment+ :metabase.measures.schema/measure])+(defmulti calculate-deps*"Implementation multimethod for [[calculate-deps]]. Dispatches on entity-type keyword.Prefer calling [[calculate-deps]] which validates the return value."@@ -20,8 +44,8 @@(mu/defn calculate-deps :- ::deps.schema/upstream-deps"Calculate upstream dependencies for a single entity.Returns a map of dependency-type -> set of entity IDs."- [entity-type :- keyword?- entity]+ [entity-type :- ::deps.dependency-types/dependency-types+ entity :- Entity](calculate-deps* entity-type entity));;; ------------------------------------------------ Helpers ------------------------------------------------@@ -50,12 +74,11 @@(mu/defn- upstream-deps:dimension-mappings :- ::deps.schema/upstream-deps"Table dependencies of `dimension-mappings`, gathered from their `:table-id`s."- [dimension-mappings]+ [dimension-mappings :- [:maybe [:sequential ::lib-metric.schema/dimension-mapping]]]{:table (into #{} (keep :table-id) dimension-mappings)})(mu/defn- upstream-deps:python-transform :- ::deps.schema/upstream-deps- [{{tables :source-tables} :source :as _py-transform}- :- [:map [:source-tables {:optional true} [:sequential ::transforms-base.u/source-table-entry]]]]+ [{{tables :source-tables} :source :as _py-transform} :- ::transforms.schema/transform]{:table (into #{} (keep :table_id) tables)});; Modified implementation of documents.models.document/document-deps@@ -81,7 +104,8 @@(defmethod calculate-deps* :card[_ {query :dataset_query, dimension-mappings :dimension_mappings :as card}]{:pre [(some? query)]}- (let [base-deps (merge-with into+ (let [query (lib-be/normalize-query query)+ base-deps (merge-with into(upstream-deps:query query)(upstream-deps:dimension-mappings dimension-mappings))param-card-ids (keep #(-> % :values_source_config :card_id) (:parameters card))]@@ -91,10 +115,14 @@param-card-ids)))(defmethod calculate-deps* :transform- [_ {{:keys [query]} :source :as transform}]+ [_ transform](cond- (transforms-base.u/query-transform? transform) (upstream-deps:query query)- (transforms-base.u/python-transform? transform) (upstream-deps:python-transform transform)+ (transforms-base.u/query-transform? transform)+ (upstream-deps:query (lib-be/normalize-query (-> transform :source :query)))++ (transforms-base.u/python-transform? transform)+ (upstream-deps:python-transform transform)+:else (do (log/warnf "Don't know how to analyze the deps of Transform %d with source type '%s'"(:id transform) (-> transform :source :type)){})))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/core.clj b/enterprise/backend/src/metabase_enterprise/dependencies/core.cljindex ce9a0411a92..45d04b5c9b8 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/core.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/core.clj@@ -7,6 +7,7 @@[metabase-enterprise.dependencies.db :as dependencies.db][metabase-enterprise.dependencies.metadata-provider :as deps.provider][metabase-enterprise.dependencies.models.dependency :as deps.graph]+ [metabase.graph.core :as graph][metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]@@ -22,8 +23,11 @@[:enum :card :transform :snippet :table])(mr/def ::updates-map- ;; TODO: Make this more specific.- [:map-of ::entity-type [:sequential [:map [:id {:optional true} :int]]]])+ [:map {:closed true}+ [:card {:optional true} [:sequential ::lib.schema.metadata/card]]+ [:transform {:optional true} [:sequential ::lib.schema.metadata/transform]]+ [:snippet {:optional true} [:sequential ::lib.schema.metadata/native-query-snippet]]+ [:table {:optional true} [:sequential ::lib.schema.metadata/table]]])(defn- transitive-dependents[& {:keys [graph updated-entities include-native?]}]@@ -46,7 +50,12 @@`include-native` will determine whether native dependents are included in the metadata provider."([base-provider :- ::lib.schema.metadata/metadata-providerupdated-entities :- ::updates-map- & {:keys [graph dependents include-native?]}]+ & {:keys [graph dependents include-native?]}+ :- [:maybe+ [:map {:closed true}+ [:graph {:optional true} [:maybe ::graph/graph]]+ [:dependents {:optional true} [:maybe [:map-of ::entity-type [:set :int]]]]+ [:include-native? {:optional true} [:maybe :boolean]]]]];; Reusing the cache with different overrides breaks the caching of [[lib.metadata/card]] calls.(lib.metadata.protocols/clear-cache! base-provider)(let [dependents (or dependents (transitive-dependents :graph graph@@ -115,7 +124,12 @@When `include-native?` is false, this function will ignore any entities using native sql and their children."([edits :- ::updates-map- & {:keys [base-provider graph include-native?]}]+ & {:keys [base-provider graph include-native?]}+ :- [:maybe+ [:map {:closed true}+ [:base-provider {:optional true} [:maybe ::lib.schema.metadata/metadata-provider]]+ [:graph {:optional true} [:maybe ::graph/graph]]+ [:include-native? {:optional true} [:maybe :boolean]]]]](let [valid-edits (if include-native?edits(into {}diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/metadata_update.clj b/enterprise/backend/src/metabase_enterprise/dependencies/metadata_update.cljindex f83d9f0cc5f..2e41ede9bb8 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/metadata_update.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/metadata_update.clj@@ -12,10 +12,12 @@[metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]+ [metabase.lib.schema.common :as lib.schema.common][metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.metadata :as lib.schema.metadata][metabase.premium-features.core :as premium-features][metabase.queries.core :as queries]+ [metabase.queries.schema :as queries.schema][metabase.util :as u][metabase.util.malli :as mu][metabase.util.malli.registry :as mr]@@ -36,7 +38,7 @@(mu/defn- dependent-mbql-cards :- [:sequential ::lib.schema.id/card]"Returns a list of all card dependencies in the transitive children of `[start-type start-id]` using `graph`."- [graph+ [graph :- ::graph/graphstart-type :- ::deps.dependency-types/dependency-typesstart-id :- ::deps.dependency-types/entity-id](let [start [start-type start-id]]@@ -46,7 +48,18 @@(not= node start))node-id))))))-(mr/def ::column-metadata-edits [:map-of :keyword :any])+(mr/def ::column-metadata-edits+ "Edits `card-metadata-edits` reports: the keys are QP-results-cased (plain keys snake_case, `:lib/...` keys stay+ kebab-case), since values are looked up straight out of `qp-results-cased-col` maps."+ [:map {:closed true}+ [:id {:optional true} [:maybe ::lib.schema.id/field]]+ [:description {:optional true} [:maybe :string]]+ [:display_name {:optional true} [:maybe :string]]+ [:semantic_type {:optional true} [:maybe ::lib.schema.common/semantic-or-relation-type]]+ [:fk_target_field_id {:optional true} [:maybe ::lib.schema.id/field]]+ [:settings {:optional true} [:maybe ::lib.schema.common/visualization-settings]]+ [:visibility_type {:optional true} [:maybe ::lib.schema.metadata/column.visibility-type]]+ [:lib/source-display-name {:optional true} [:maybe :string]]])(mr/def ::card-metadata-edits [:map-of :string ::column-metadata-edits])(mr/def ::card-list-metadata-edits [:map-of ::lib.schema.id/card ::card-metadata-edits])@@ -119,7 +132,7 @@[original-mp :- ::lib.schema.metadata/metadata-providerablestart-type :- ::deps.dependency-types/dependency-typesstart-id :- ::deps.dependency-types/entity-id- previous-object :- :any+ previous-object :- ::queries.schema/cardmetadata-type :- :keyword];; Notes on metadata providers:;;diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding_error.clj b/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding_error.cljindex cf097f37590..9ba72bdefd0 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding_error.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding_error.clj@@ -19,7 +19,7 @@(mr/def ::analysis-finding-error"Schema for an AnalysisFindingError record."- [:map+ [:map {:closed true}[:id ms/PositiveInt][:analyzed_entity_type ::deps.dependency-types/dependency-types][:analyzed_entity_id ms/PositiveInt]@@ -35,7 +35,7 @@(mr/def ::error-input"Schema for error input maps passed to replace-errors-for-entity!"- [:map+ [:map {:closed true}[:error-type ::lib.schema.validate/validate-error-type][:error-detail {:optional true} [:maybe :string]][:source-entity-type {:optional true} [:maybe ::lib.schema.validate/source-entity-type]]diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.cljindex 8b228b4068f..5897873b179 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj@@ -3,9 +3,17 @@[clojure.set :as set][metabase-enterprise.dependencies.db :as dependencies.db][metabase-enterprise.dependencies.dependency-types :as deps.dependency-types]+ [metabase-enterprise.sandbox.schema]+ [metabase.documents.schema :as documents.schema][metabase.graph.core :as graph][metabase.lib.core :as lib]+ [metabase.lib.schema.metadata :as lib.schema.metadata]+ [metabase.measures.schema][metabase.models.interface :as mi]+ [metabase.native-query-snippets.schema]+ [metabase.queries.schema :as queries.schema]+ [metabase.segments.schema]+ [metabase.transforms.schema :as transforms.schema][metabase.util :as u][metabase.util.malli :as mu][methodical.core :as methodical]@@ -178,7 +186,22 @@(mu/defn is-native-entity? :- :boolean"Checks whether an entity involves native sql. `entity` can either be a toucan object or a metadata object."[entity-type :- ::deps.dependency-types/dependency-types- entity]+ entity :- [:multi {:dispatch (fn [x] (if (:lib/type x) :lib-metadata :row))}+ [:lib-metadata [:multi {:dispatch :lib/type}+ [:metadata/card ::lib.schema.metadata/card]+ [:metadata/transform ::lib.schema.metadata/transform]+ [:metadata/native-query-snippet ::lib.schema.metadata/native-query-snippet]+ [:metadata/table ::lib.schema.metadata/table]]]+ [:row [:or+ ::queries.schema/card+ :metabase.warehouse-schema.schema/table+ :metabase.native-query-snippets.schema/native-query-snippet+ ::transforms.schema/transform+ :metabase.dashboards.schema/dashboard+ ::documents.schema/document+ :metabase-enterprise.sandbox.schema/sandbox+ :metabase.segments.schema/segment+ :metabase.measures.schema/measure]]]](boolean(case entity-type:card (some-> entitydiff --git a/enterprise/backend/src/metabase_enterprise/dependencies/schema.clj b/enterprise/backend/src/metabase_enterprise/dependencies/schema.cljindex 7826c3b9257..219f8caaad5 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/schema.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/schema.clj@@ -18,14 +18,10 @@(mr/def ::analysis-finding"A AnalysisFinding as selected from the app DB: every column of `:analysis_finding`."- [:map {:closed true}- [:id ms/PositiveInt]- [:analyzed_entity_type [:or :keyword :string]]- [:analyzed_entity_id ms/PositiveInt]- [:analysis_version :int]- [:analyzed_at ms/TemporalInstant]- [:result :boolean]- [:stale :boolean]])+ [:merge+ ::analysis-finding.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::analysis-finding.update"What an update (or insert) of a AnalysisFinding accepts: every column of `:analysis_finding` except `id`, all optional."@@ -39,14 +35,10 @@(mr/def ::analysis-finding-error"A AnalysisFindingError as selected from the app DB: every column of `:analysis_finding_error`."- [:map {:closed true}- [:id ms/PositiveInt]- [:analyzed_entity_type [:or :keyword :string]]- [:analyzed_entity_id ms/PositiveInt]- [:error_type [:or :keyword :string]]- [:error_detail [:maybe :string]]- [:source_entity_type [:maybe [:or :keyword :string]]]- [:source_entity_id [:maybe ms/PositiveInt]]])+ [:merge+ ::analysis-finding-error.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::analysis-finding-error.update"What an update (or insert) of a AnalysisFindingError accepts: every column of `:analysis_finding_error` except `id`, all optional."@@ -60,12 +52,10 @@(mr/def ::dependency"A Dependency as selected from the app DB: every column of `:dependency`."- [:map {:closed true}- [:id ms/PositiveInt]- [:from_entity_type [:or :keyword :string]]- [:from_entity_id ms/PositiveInt]- [:to_entity_type [:or :keyword :string]]- [:to_entity_id ms/PositiveInt]])+ [:merge+ ::dependency.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::dependency.update"What an update (or insert) of a Dependency accepts: every column of `:dependency` except `id`, all optional."@@ -77,15 +67,10 @@(mr/def ::dependency-status"A DependencyStatus as selected from the app DB: every column of `:dependency_status`."- [:map {:closed true}- [:id ms/PositiveInt]- [:entity_type [:or :keyword :string]]- [:entity_id ms/PositiveInt]- [:dependency_analysis_version :int]- [:stale :boolean]- [:fail_count :int]- [:next_retry_at [:maybe ms/TemporalInstant]]- [:terminal :boolean]])+ [:merge+ ::dependency-status.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::dependency-status.update"What an update (or insert) of a DependencyStatus accepts: every column of `:dependency_status` except `id`, all optional."diff --git a/enterprise/backend/src/metabase_enterprise/gsheets/api.clj b/enterprise/backend/src/metabase_enterprise/gsheets/api.cljindex 9027177a576..d24cb0307ee 100644--- a/enterprise/backend/src/metabase_enterprise/gsheets/api.clj+++ b/enterprise/backend/src/metabase_enterprise/gsheets/api.clj@@ -152,7 +152,7 @@(mu/defn- hm-delete-conn! :- :hm-client/http-reply"Delete (presumably a gdrive) connection on HM."- [conn-id]+ [conn-id :- ms/NonBlankString](hm.client/make-request :delete (str "/api/v2/mb/connections/" conn-id)))(defn- reset-gsheets-status!@@ -175,7 +175,7 @@(mu/defn hm-get-gdrive-conn :- :hm-client/http-reply"Get a specific gdrive connection by id."- [id]+ [id :- ms/NonBlankString](when-not id(throw (ex-info "Cannot fetch Google Drive connection: ID is nil" {})))(hm.client/make-request :get (str "/api/v2/mb/connections/" id)))@@ -196,7 +196,7 @@(mu/defn- hm-create-gdrive-conn! :- :hm-client/http-reply"Creating a gdrive connection on HM starts the sync w/ drive folder or sheet."- [resource-url]+ [resource-url :- ms/NonBlankString](hm.client/make-request :post "/api/v2/mb/connections" {:type (url-type resource-url) :secret {:resources [resource-url]}}));;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -343,7 +343,7 @@(mu/defn- hm-sync-conn! :- :hm-client/http-reply"Sync a (presumably a gdrive) connection on HM."- [conn-id]+ [conn-id :- ms/NonBlankString](hm.client/make-request :put (str "/api/v2/mb/connections/" conn-id "/sync")));; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers todiff --git a/enterprise/backend/src/metabase_enterprise/harbormaster/client.clj b/enterprise/backend/src/metabase_enterprise/harbormaster/client.cljindex 9c9762c6bac..b7cf7eb60d2 100644--- a/enterprise/backend/src/metabase_enterprise/harbormaster/client.clj+++ b/enterprise/backend/src/metabase_enterprise/harbormaster/client.clj@@ -23,7 +23,7 @@(cond->> s(not (str/starts-with? s "/")) (str "/")))-(mu/defn- ->requestor [method]+(mu/defn- ->requestor [method :- [:enum :get :head :post :put :delete :options :copy :move :patch]](case method:get http/get:head http/head@@ -36,9 +36,9 @@:patch http/patch))(mu/defn- get-safe-status- [response]- (when (number? (:status response))- (http/success? response)))+ [status :- [:maybe :int]]+ (when (number? status)+ (http/success? {:status status})))(defn- ->config"Returns the config needed to call [[make-request]].@@ -93,7 +93,7 @@(defn- calculate-success [response url request](try- (get-safe-status response)+ (get-safe-status (:status response))(catch Exception e(log/errorf "Error decoding response from %s, is it json? %s" url (ex-message e)){:response response@@ -110,7 +110,10 @@Returns a tuple of [:ok response] if the request was successful, or [:error response] if it failed."[method :- [:enum :get :head :post :put :delete :options :copy :move :patch]url :- :string- & [body]]+ & [body] :- [:* [:map {:closed true}+ [:type :string]+ [:secret [:map {:closed true}+ [:resources [:sequential :string]]]]]]](let [{:keys [store-api-urlapi-key]} (->config)request (cond-> {:headers {"Authorization" (str "Bearer " api-key)diff --git a/enterprise/backend/src/metabase_enterprise/impersonation/schema.clj b/enterprise/backend/src/metabase_enterprise/impersonation/schema.cljindex e99afe45fcd..6c2f8203855 100644--- a/enterprise/backend/src/metabase_enterprise/impersonation/schema.clj+++ b/enterprise/backend/src/metabase_enterprise/impersonation/schema.clj@@ -7,11 +7,10 @@(mr/def ::connection-impersonation"A ConnectionImpersonation as selected from the app DB: every column of `:connection_impersonations`."- [:map {:closed true}- [:id ms/PositiveInt]- [:db_id ::lib.schema.id/database]- [:group_id ms/PositiveInt]- [:attribute [:maybe :string]]])+ [:merge+ ::connection-impersonation.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::connection-impersonation.update"What an update (or insert) of a ConnectionImpersonation accepts: every column of `:connection_impersonations` except `id`, all optional."diff --git a/enterprise/backend/src/metabase_enterprise/metabot/schema.clj b/enterprise/backend/src/metabase_enterprise/metabot/schema.cljindex a432cd5eda9..cdff509152b 100644--- a/enterprise/backend/src/metabase_enterprise/metabot/schema.clj+++ b/enterprise/backend/src/metabase_enterprise/metabot/schema.clj@@ -6,10 +6,10 @@(mr/def ::metabot-group-limit"A MetabotGroupLimit as selected from the app DB: every column of `:metabot_group_limit`."- [:map {:closed true}- [:id ms/PositiveInt]- [:group_id ms/PositiveInt]- [:max_usage :int]])+ [:merge+ ::metabot-group-limit.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::metabot-group-limit.update"What an update (or insert) of a MetabotGroupLimit accepts: every column of `:metabot_group_limit` except `id`, all optional."@@ -19,10 +19,10 @@(mr/def ::metabot-instance-limit"A MetabotInstanceLimit as selected from the app DB: every column of `:metabot_instance_limit`."- [:map {:closed true}- [:id ms/PositiveInt]- [:tenant_id [:maybe ms/PositiveInt]]- [:max_usage [:maybe :int]]])+ [:merge+ ::metabot-instance-limit.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::metabot-instance-limit.update"What an update (or insert) of a MetabotInstanceLimit accepts: every column of `:metabot_instance_limit` except `id`, all optional."@@ -32,11 +32,10 @@(mr/def ::metabot-permissions"A MetabotPermissions as selected from the app DB: every column of `:metabot_permissions`."- [:map {:closed true}- [:id ms/PositiveInt]- [:group_id ms/PositiveInt]- [:perm_type [:or :keyword :string]]- [:perm_value [:or :keyword :string]]])+ [:merge+ ::metabot-permissions.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::metabot-permissions.update"What an update (or insert) of a MetabotPermissions accepts: every column of `:metabot_permissions` except `id`, all optional."diff --git a/enterprise/backend/src/metabase_enterprise/mfa/db.clj b/enterprise/backend/src/metabase_enterprise/mfa/db.cljindex bf637bd8783..8e9981cb6d7 100644--- a/enterprise/backend/src/metabase_enterprise/mfa/db.clj+++ b/enterprise/backend/src/metabase_enterprise/mfa/db.clj@@ -94,8 +94,8 @@[:id {:optional true} ms/PositiveInt][:user_id {:optional true} [:maybe ::lib.schema.id/user]][:provider {:optional true} [:maybe [:or :keyword :string]]]- [:credentials {:optional true} [:maybe :map]]- [:metadata {:optional true} [:maybe :map]]+ [:credentials {:optional true} [:maybe ::auth-identity.schema/auth-identity.credentials.totp]]+ [:metadata {:optional true} [:maybe [:map {:closed true}]]][:provider_id {:optional true} [:maybe :string]][:last_used_at {:optional true} [:maybe ms/TemporalInstant]][:expires_at {:optional true} [:maybe ms/TemporalInstant]]diff --git a/enterprise/backend/src/metabase_enterprise/permission_debug/impl.clj b/enterprise/backend/src/metabase_enterprise/permission_debug/impl.cljindex 7260f1c7220..7590f8e51de 100644--- a/enterprise/backend/src/metabase_enterprise/permission_debug/impl.clj+++ b/enterprise/backend/src/metabase_enterprise/permission_debug/impl.clj@@ -28,13 +28,16 @@"Format of the response the Debugger will return"[:schema {:registry{::group-id :int- ::perm-debug-info [:map+ ::perm-debug-info [:map {:closed true}[:model-type [:enum "card" "invalid"]][:model-id :string][:decision [:enum "allow" "denied" "limited"]][:segment [:set [:enum "sandboxed" "impersonated" "routed"]]][:message [:sequential :string]]- [:data [:map]]+ [:data [:map {:closed true}+ [:blocked-tables {:optional true} [:map-of :string [:set :string]]]+ [:download-limited-tables {:optional true} [:map-of :string [:set :string]]]+ [:download-no-tables {:optional true} [:map-of :string [:set :string]]]]][:suggestions [:map-of ::group-id :string]]]}}::perm-debug-info])diff --git a/enterprise/backend/src/metabase_enterprise/premium_features/airgap.clj b/enterprise/backend/src/metabase_enterprise/premium_features/airgap.cljindex 2af954630f9..16e960aa6a0 100644--- a/enterprise/backend/src/metabase_enterprise/premium_features/airgap.clj+++ b/enterprise/backend/src/metabase_enterprise/premium_features/airgap.clj@@ -27,7 +27,7 @@(mu/defn- decode-token :- :map"Given an encrypted airgap token, decrypts it and returns a TokenStatus"- [token]+ [token :- :string](when-not (token? token)(throw (ex-info "Malformed airgap token" {:token token})))(let [token (str/replace token #"^airgap_" "")diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/db.cljindex a204cffd510..796e3fadc14 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj@@ -11,9 +11,14 @@[metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay][toucan2.core :as t2]))+(def ^:private ConditionKey+ "The column keys used in the `:conditions` / `:cascade-filter` / `:removal-conditions` of a remote-sync model+ spec (see `metabase-enterprise.remote-sync.spec`)."+ [:enum :exploration_id :built_in_type :active :entity_id :collection_id :archived :archived_at])+(def ^:private Conditions"A map of column to value (possibly nil) or Toucan 2 operator-vector value, or nil for none."- [:maybe [:map-of :keyword [:maybe [:or :string :int :boolean :keyword sequential?]]]])+ [:maybe [:map-of ConditionKey [:maybe [:or :string :int :boolean :keyword sequential?]]]])(def ^:private RemovalOpts"The `:scope-key`, `:synced-collection-ids`, `:entity-ids`, and `:removal-conditions` describing which rows an@@ -28,11 +33,11 @@"A `:model_id`: the primary key of the referenced entity, or the `-1` sentinel(`metabase-enterprise.remote-sync.settings/transforms-root-id`) standing in for the virtual Transforms rootCollection."- [:or ms/PositiveInt [:= -1]])+ [:maybe [:or ms/PositiveInt [:= -1]]])(def ^:private Path"A `{:db_name :schema :table_name :field_name}` path used to locate a Table or Field."- [:map+ [:map {:closed true}[:db_name :string][:schema {:optional true} [:maybe :string]][:table_name :string]@@ -597,8 +602,8 @@(t2/delete! :model/RemoteSyncObject :model_type model-type :model_id [:in model-ids]))(mu/defn delete-rsos-of-keys!- "Delete the RemoteSyncObjects keyed by the `:model_type`/`:model_id` of `rows` (other keys are ignored)."- [rows :- [:sequential [:map [:model_type :string] [:model_id ms/PositiveInt]]]]+ "Delete the RemoteSyncObjects keyed by the `:model_type`/`:model_id` of `rows`."+ [rows :- [:sequential ::remote-sync.schema/remote-sync-object.update]](t2/delete! :model/RemoteSyncObject {:where (rso-keys-expr rows)}))(mu/defn delete-all-rsos!diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.cljindex e381a3269cf..e2907f39260 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj@@ -469,11 +469,12 @@:let [model-key (:model-key (spec/spec-for-model-type model_type))eid (when model-key (remote-sync.db/entity-id model-key model_id))]:when (not (loaded-eid? model_type eid))]- {:model_type model_type :model_id model_id :model-key model-key})+ {:model_type model_type :model_id model_id})+ model-key-of (fn [{:keys [model_type]}] (:model-key (spec/spec-for-model-type model_type)))sync-rows (spec/sync-all-entities! sync-timestamp imported-data)](remote-sync.task/update-progress! task-id 0.8)(t2/with-transaction [_conn]- (doseq [[model-key ds] (group-by :model-key deletes)]+ (doseq [[model-key ds] (group-by model-key-of deletes)](remote-sync.db/delete-instances! model-key (mapv :model_id ds)))(when (seq deletes)(remote-sync.db/delete-rsos-of-keys! deletes))@@ -485,7 +486,7 @@;; We skip the whole-appdb reindex the full load runs. Added/modified entities are already;; re-indexed by the load itself — serdes' t2 insert!/update! fire the :hook/search-index;; after-insert/after-update hooks. Deletes have no such hook, so remove them explicitly.- (doseq [[model-key ds] (group-by :model-key deletes)]+ (doseq [[model-key ds] (group-by model-key-of deletes)](search/delete! model-key (mapv :model_id ds)))(remote-sync.task/update-progress! task-id 0.95)(log/info "Successfully reloaded entities from git repository")diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_task.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_task.cljindex e1b2d6fc579..07fb43303d4 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_task.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_task.clj@@ -3,6 +3,7 @@(:require[java-time.api :as t][metabase-enterprise.remote-sync.db :as remote-sync.db]+ [metabase-enterprise.remote-sync.schema :as remote-sync.schema][metabase.models.interface :as mi][metabase.settings.core :as setting][metabase.util.malli :as mu]@@ -45,8 +46,8 @@Throws ExceptionInfo if a running task already exists."[sync-task-type :- ::remote-sync-task-type- user-id :- [:maybe pos-int?] &- [additional-fields :- [:map]]]+ user-id :- [:maybe pos-int?]+ & [additional-fields] :- [:* ::remote-sync.schema/remote-sync-task.update]](remote-sync.db/insert-task! (merge {:sync_task_type sync-task-type:initiated_by user-id:progress 0}diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/schema.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/schema.cljindex a45c869222e..292d5de062d 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/schema.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/schema.clj@@ -171,19 +171,10 @@(mr/def ::remote-sync-object"A RemoteSyncObject as selected from the app DB: every column of `:remote_sync_object`."- [:map {:closed true}- [:id ms/PositiveInt]- [:model_type [:or :keyword :string]]- [:model_id [:maybe :int]]- [:status [:or :keyword :string]]- [:status_changed_at ms/TemporalInstant]- [:model_name :string]- [:model_collection_id [:maybe ::lib.schema.id/collection]]- [:model_display [:maybe [:or :keyword :string]]]- [:model_table_id [:maybe ::lib.schema.id/table]]- [:model_table_name [:maybe :string]]- [:file_path [:maybe :string]]- [:content_hash [:maybe :string]]])+ [:merge+ ::remote-sync-object.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::remote-sync-object.update"What an update (or insert) of a RemoteSyncObject accepts: every column of `:remote_sync_object` except `id`, all optional."@@ -202,23 +193,31 @@(mr/def ::remote-sync-task.outcome"The `:outcome` column of a RemoteSyncTask, decoded."- :map)+ [:multi {:dispatch :kind}+ ["pulled" [:map {:closed true}+ [:kind [:= "pulled"]]+ [:count :int]+ [:branch [:maybe :string]]]]+ ["pull-skipped" [:map {:closed true}+ [:kind [:= "pull-skipped"]]]]+ ["pushed" [:map {:closed true}+ [:kind [:= "pushed"]]+ [:count :int]+ [:branch [:maybe :string]]]]+ ["push-skipped" [:map {:closed true}+ [:kind [:= "push-skipped"]]]]+ ["merged" [:map {:closed true}+ [:kind [:= "merged"]]+ [:pulled :int]+ [:pushed :int]+ [:branch [:maybe :string]]]]])(mr/def ::remote-sync-task"A RemoteSyncTask as selected from the app DB: every column of `:remote_sync_task`."- [:map {:closed true}- [:id ms/PositiveInt]- [:sync_task_type [:or :keyword :string]]- [:progress [:maybe number?]]- [:cancelled :boolean]- [:started_at ms/TemporalInstant]- [:ended_at [:maybe ms/TemporalInstant]]- [:last_progress_report_at ms/TemporalInstant]- [:initiated_by [:maybe ::lib.schema.id/user]]- [:error_message [:maybe :string]]- [:version [:maybe :string]]- [:conflicts [:maybe [:sequential :string]]]- [:outcome [:maybe ::remote-sync-task.outcome]]])+ [:merge+ ::remote-sync-task.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::remote-sync-task.update"What an update (or insert) of a RemoteSyncTask accepts: every column of `:remote_sync_task` except `id`, all optional."@@ -232,5 +231,5 @@[:initiated_by {:optional true} [:maybe ::lib.schema.id/user]][:error_message {:optional true} [:maybe :string]][:version {:optional true} [:maybe :string]]- [:conflicts {:optional true} [:maybe [:sequential :string]]]+ [:conflicts {:optional true} [:maybe [:or [:sequential :string] [:set :string]]]][:outcome {:optional true} [:maybe ::remote-sync-task.outcome]]])diff --git a/enterprise/backend/src/metabase_enterprise/replacement/schema.clj b/enterprise/backend/src/metabase_enterprise/replacement/schema.cljindex feedc407c82..1f2dc636f8f 100644--- a/enterprise/backend/src/metabase_enterprise/replacement/schema.clj+++ b/enterprise/backend/src/metabase_enterprise/replacement/schema.clj@@ -13,6 +13,9 @@(mr/def ::source-entity-type[:enum :card :table :transform])+(mr/def ::source-ref+ [:tuple ::source-entity-type ::source-entity-id])+(mr/def ::run-idpos-int?)@@ -60,19 +63,10 @@(mr/def ::replacement-run"A ReplacementRun as selected from the app DB: every column of `:source_replacement_run`."- [:map {:closed true}- [:id ms/PositiveInt]- [:source_entity_type [:or :keyword :string]]- [:source_entity_id ms/PositiveInt]- [:target_entity_type [:or :keyword :string]]- [:target_entity_id ms/PositiveInt]- [:status [:or :keyword :string]]- [:is_active [:maybe :boolean]]- [:progress [:maybe number?]]- [:message [:maybe :string]]- [:user_id [:maybe ::lib.schema.id/user]]- [:start_time ms/TemporalInstant]- [:end_time [:maybe ms/TemporalInstant]]])+ [:merge+ ::replacement-run.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::replacement-run.update"What an update (or insert) of a ReplacementRun accepts: every column of `:source_replacement_run` except `id`, all optional."diff --git a/enterprise/backend/src/metabase_enterprise/replacement/source_check.clj b/enterprise/backend/src/metabase_enterprise/replacement/source_check.cljindex 185464bbc30..d2e3f302b8c 100644--- a/enterprise/backend/src/metabase_enterprise/replacement/source_check.clj+++ b/enterprise/backend/src/metabase_enterprise/replacement/source_check.clj@@ -59,8 +59,8 @@"Check whether `old-source` can be replaced by `new-source`. Returns a map with`:success`, `:errors` (unique top-level error types), and `:column_mappings`.Arguments match `swap-source`: each is a `[entity-type entity-id]` pair."- [[old-type old-id :as old-ref]- [new-type new-id :as new-ref]]+ [[old-type old-id :as old-ref] :- ::replacement.schema/source-ref+ [new-type new-id :as new-ref] :- ::replacement.schema/source-ref](if (= old-ref new-ref){:success false}(let [source-db-id (fetch-source-database-id old-type old-id)diff --git a/enterprise/backend/src/metabase_enterprise/replacement/util.clj b/enterprise/backend/src/metabase_enterprise/replacement/util.cljindex ead0ab04466..0e2db2118d8 100644--- a/enterprise/backend/src/metabase_enterprise/replacement/util.clj+++ b/enterprise/backend/src/metabase_enterprise/replacement/util.clj@@ -1,5 +1,6 @@(ns metabase-enterprise.replacement.util(:require+ [metabase.lib.schema :as lib.schema][metabase.util.malli :as mu]))(set! *warn-on-reflection* true)@@ -8,5 +9,5 @@"Returns true if `maybe-query` has at least one stage.This is used to skip fully broken queries in the app db."- [maybe-query :- [:maybe :map]]+ [maybe-query :- [:maybe [:or ::lib.schema/query [:map {:closed true}]]]](some? (seq (:stages maybe-query))))diff --git a/enterprise/backend/src/metabase_enterprise/replacement/walk.clj b/enterprise/backend/src/metabase_enterprise/replacement/walk.cljindex 2e40e8d21c5..83a7ea1bf36 100644--- a/enterprise/backend/src/metabase_enterprise/replacement/walk.clj+++ b/enterprise/backend/src/metabase_enterprise/replacement/walk.clj@@ -6,20 +6,22 @@[metabase.lib.schema.parameter :as lib.schema.parameter][metabase.lib.util :as lib.util][metabase.models.visualization-settings :as vs]- [metabase.util.malli :as mu]))+ [metabase.parameters.schema :as parameters.schema]+ [metabase.util.malli :as mu]+ [metabase.util.malli.schema :as ms]))(set! *warn-on-reflection* true)(mu/defn parameter-source-card-ids :- [:set ::lib.schema.id/card]"Get all card IDs referenced by `:values_source_config` in the given parameters."- [parameters :- [:sequential :map]]+ [parameters :- [:sequential ::parameters.schema/parameter]](into #{} (keep #(-> % :values_source_config :card_id)) parameters))(mu/defn walk-parameter-source-card-ids :- [:sequential :map]"Walk the parameters and update the card IDs in `:values_source_config` using the provided function.`card-id-fn` will be called with a card ID and should return a new card ID."- [parameters :- [:sequential :map]+ [parameters :- [:sequential ::parameters.schema/parameter]card-id-fn :- fn?](mapv #(m/update-existing-in % [:values_source_config :card_id] card-id-fn) parameters))@@ -28,7 +30,7 @@in the `:values_source_config` using the provided function.`ref-fn` will be called with a ref and a card ID and should return a new ref."- [parameters :- [:sequential :map]+ [parameters :- [:sequential ::parameters.schema/parameter]ref-fn :- fn?](letfn [(update-legacy-ref [legacy-ref card-id](try@@ -47,14 +49,14 @@(mu/defn parameter-mapping-card-ids :- [:set ::lib.schema.id/card]"Get all card IDs referenced by the parameter mappings."- [parameter-mappings :- [:sequential :map]]+ [parameter-mappings :- [:maybe [:sequential ::parameters.schema/parameter-mapping]]](into #{} (keep :card_id) parameter-mappings))(mu/defn walk-parameter-mapping-targets :- [:sequential :map]"Walk the parameter mappings and update the targets using the provided function.`target-fn` will be called with a parameter target and a card ID and should return a new parameter target."- [parameter-mappings :- [:sequential :map]+ [parameter-mappings :- [:maybe [:sequential ::parameters.schema/parameter-mapping]]target-fn :- fn?](mapv (fn [mapping](or (when-some [card-id (:card_id mapping)]@@ -67,7 +69,7 @@"Walk the viz settings and update the refs using the provided function.`ref-fn` will be called with a ref and should return either a string to be used as a column name."- [viz-settings :- :map+ [viz-settings :- ms/VisualizationSettingsref-fn :- fn?](letfn [(update-legacy-ref-or-name [ref-or-name](or (when (vector? ref-or-name)@@ -106,13 +108,13 @@(mu/defn click-behavior-card-id :- [:maybe ::lib.schema.id/card]"Returns the card ID from a click behavior if it links to a card."- [click-behavior :- :map]+ [click-behavior :- ms/VisualizationSettings](when (= ::vs/card (::vs/link-type click-behavior))(::vs/link-target-id click-behavior)))(mu/defn viz-settings-click-behavior-card-ids :- [:set ::lib.schema.id/card]"Get all card IDs referenced by the click behaviors in the viz settings."- [viz-settings :- :map]+ [viz-settings :- ms/VisualizationSettings](into #{}(concat;; global click behavior (non-table cards)@@ -127,7 +129,7 @@"Walk the click behaviors in the viz settings and update the targets using the provided function.`target-fn` will be called with a parameter target and a card ID and should return a new parameter target."- [viz-settings :- :map+ [viz-settings :- ms/VisualizationSettingstarget-fn :- fn?](letfn [(update-mapping [card-id mapping](or (when-some [target (some->> mapping ::vs/param-mapping-target ::vs/param-dimensiondiff --git a/enterprise/backend/src/metabase_enterprise/sandbox/api/column_filter.clj b/enterprise/backend/src/metabase_enterprise/sandbox/api/column_filter.cljindex 68083a3b84a..0ee4f02d349 100644--- a/enterprise/backend/src/metabase_enterprise/sandbox/api/column_filter.clj+++ b/enterprise/backend/src/metabase_enterprise/sandbox/api/column_filter.clj@@ -26,9 +26,34 @@[metabase.lib.core :as lib][metabase.permissions.core :as perms][metabase.premium-features.core :refer [defenterprise]]+ [metabase.queries.schema :as queries.schema][metabase.util :as u][metabase.util.malli :as mu]- [metabase.util.malli.schema :as ms]))+ [metabase.util.malli.schema :as ms]+ [metabase.warehouse-schema.schema :as warehouse-schema.schema]))++(def ^:private Card+ ::queries.schema/card)++(def ^:private PartialField+ "A Field, as hydrated onto another Field's `:target`/`:name_field` -- only some columns are fetched for these,+ so it's not a full `::warehouse-schema.schema/field`."+ [:merge+ ::warehouse-schema.schema/field.update+ [:map {:closed true}+ [:id {:optional true} ms/PositiveInt]]])++(def ^:private Field+ [:or+ [:merge+ ::warehouse-schema.schema/field+ [:map {:closed true}+ [:target {:optional true} [:maybe PartialField]]+ [:dimensions {:optional true} [:sequential ::warehouse-schema.schema/dimension]]+ [:name_field {:optional true} [:maybe PartialField]]]]+ [:map {:closed true}+ [:id {:optional true} ms/PositiveInt]+ [:name {:optional true} :string]]])(mu/defn find-sandbox-source-cards :- [:map-of ms/PositiveInt :map]"Return `{table-id => sandbox-source-card}` for the `table-ids` that have a Card-backed sandbox for the current user.@@ -56,8 +81,8 @@"Filter `fields` to those exposed by `card`, by field id for MBQL sandboxes and by name for native ones.Returns `fields` unchanged when `card` is nil or has no `:dataset_query`, and an empty seq when its`:result_metadata` is nil/empty (fail-closed; see ns docstring)."- [card :- [:maybe :map]- fields :- [:sequential :map]]+ [card :- [:maybe Card]+ fields :- [:sequential Field]](let [sandbox-query (:dataset_query card)](cond(nil? card) fields@@ -73,7 +98,7 @@"Return the `fields` for `table-id` that are visible to the current user under their sandbox configuration.Non-sandboxed tables and superusers receive `fields` unchanged."[table-id :- ms/PositiveInt- fields :- [:sequential :map]]+ fields :- [:sequential Field]](if-let [card (get (find-sandbox-source-cards #{table-id}) table-id)](filter-fields-by-card card fields)fields))@@ -81,7 +106,7 @@(mu/defn batch-filter-fields-by-table"Return the `{table-id => fields}` map with each table's fields filtered to those visible to the current user.Filters per the user's sandbox configuration, using a single DB query for all sandbox source cards."- [fields-by-table :- [:map-of ms/PositiveInt [:sequential :map]]]+ [fields-by-table :- [:map-of ms/PositiveInt [:sequential Field]]](let [card-by-table (find-sandbox-source-cards (set (keys fields-by-table)))](into {}(map (fn [[table-id fields]]diff --git a/enterprise/backend/src/metabase_enterprise/sandbox/db.clj b/enterprise/backend/src/metabase_enterprise/sandbox/db.cljindex 7528107c562..a4e1e9f7587 100644--- a/enterprise/backend/src/metabase_enterprise/sandbox/db.clj+++ b/enterprise/backend/src/metabase_enterprise/sandbox/db.clj@@ -94,7 +94,7 @@[:table_id ::lib.schema.id/table][:card_id {:optional true} [:maybe ::lib.schema.id/card]][:group_id ms/PositiveInt]- [:attribute_remappings {:optional true} [:maybe :map]]]]+ [:attribute_remappings {:optional true} [:maybe ::sandbox.schema/attribute-remappings]]]](first (t2/insert-returning-instances! :model/Sandbox sandbox)))(mu/defn update-sandbox!@@ -150,7 +150,7 @@(mu/defn table"The Table with `table-id`, or nil."- [table-id :- ::lib.schema.id/table]+ [table-id :- [:maybe ::lib.schema.id/table]](t2/select-one :model/Table :id table-id {:from [(warehouse-schema-overlay/table-query)]}))(mu/defn tables-of-databasediff --git a/enterprise/backend/src/metabase_enterprise/sandbox/models/sandbox.clj b/enterprise/backend/src/metabase_enterprise/sandbox/models/sandbox.cljindex fecdd733cf7..d1f8dc8bdb1 100644--- a/enterprise/backend/src/metabase_enterprise/sandbox/models/sandbox.clj+++ b/enterprise/backend/src/metabase_enterprise/sandbox/models/sandbox.clj@@ -19,6 +19,7 @@[metabase.models.interface :as mi][metabase.permissions.core :as perms][metabase.premium-features.core :refer [defenterprise]]+ [metabase.queries.schema :as queries.schema][metabase.query-processor.error-type :as qp.error-type][metabase.query-processor.preprocess :as qp.preprocess][metabase.query-processor.schema :as qp.schema]@@ -128,14 +129,15 @@"Make sure the result metadata data columns for the Card associated with a sandbox match up with the columns in the Tablethat's getting sandboxed The base types of the Card columns can derive from the respective base types of the columns inthe Table itself, but you cannot return an entirely different type. Extra columns in the sandboxing Card are ignored."- ([{card-id :card_id, table-id :table_id}]+ ([{card-id :card_id, table-id :table_id} :- [:or ::sandbox.schema/sandbox ::sandbox.schema/sandbox.update]];; not all sandboxes have Cards(when card-id;; not all Cards have saved result metadata(when-let [result-metadata (not-empty (sandbox.db/card-result-metadata card-id))](check-columns-match-table table-id result-metadata))))- ([table-id :- ::lib.schema.id/table result-metadata-columns]+ ([table-id :- ::lib.schema.id/table+ result-metadata-columns :- [:maybe ::queries.schema/card.result-metadata]](let [table-cols (table-field-names->cols table-id)](doseq [col result-metadata-columns:let [table-col (get table-cols (:name col))]@@ -220,6 +222,14 @@:user-id api/*current-user-id*})inserted-sandbox)))))+(defn- normalize-sandbox-attribute-remappings+ "Normalize `:attribute_remappings` on a Sandbox map, if present. `before-insert`/`before-update` hooks see this+ column in whatever raw, not-yet-normalized shape the caller supplied it in."+ [sandbox-like]+ (cond-> sandbox-like+ (contains? sandbox-like :attribute_remappings)+ (update :attribute_remappings sandbox.schema/normalize-attribute-remappings)))+(t2/define-before-insert :model/Sandbox[{:keys [table_id group_id], :as gtap}](let [db-id (database/table-id->database-id table_id)]@@ -227,7 +237,7 @@(when (= (perms/table-permission-for-groups #{group_id} :perms/create-queries db-id table_id) :query-builder-and-native)(perms/set-database-permission! group_id db-id :perms/create-queries :query-builder)))(u/prog1 gtap- (check-columns-match-table gtap)))+ (check-columns-match-table (normalize-sandbox-attribute-remappings gtap))))(t2/define-before-update :model/Sandbox[{:keys [id], :as updates}]@@ -239,4 +249,4 @@{:id id:status-code 400})))(when (:card_id updates)- (check-columns-match-table updated)))))+ (check-columns-match-table (normalize-sandbox-attribute-remappings updated))))))diff --git a/enterprise/backend/src/metabase_enterprise/sandbox/query_processor/middleware/sandboxing.clj b/enterprise/backend/src/metabase_enterprise/sandbox/query_processor/middleware/sandboxing.cljindex 493ded4fc51..4d11a5c33a3 100644--- a/enterprise/backend/src/metabase_enterprise/sandbox/query_processor/middleware/sandboxing.clj+++ b/enterprise/backend/src/metabase_enterprise/sandbox/query_processor/middleware/sandboxing.clj@@ -43,16 +43,19 @@;;; from [[metabase-enterprise.sandbox.api.util/enforced-sandboxes-for-tables]] for consistency with all of the rest;;; of the QP code. Or maybe add this to the Metadata Provider (or a special "Enterprise" Metadata Provider)?(mr/def ::sandbox- [:map+ [:map {:closed true}+ [:id {:optional true} [:maybe :int]]+ [:group_id {:optional true} [:maybe :int]][:table_id ::lib.schema.id/table][:card_id {:optional true} [:maybe ::lib.schema.id/card]]+ [:table {:optional true} [:maybe :metabase.warehouse-schema.schema/table]][:attribute_remappings {:optional true} [:maybe[:map-of#_attribute-name ::lib.schema.common/non-blank-string#_target ::lib.schema.parameter/target]]]])(mu/defn- query->all-table-ids :- [:maybe [:set ::lib.schema.id/table]]- [query]+ [query :- ::lib.schema/query](u/prog1 (lib/all-source-table-ids query)(when (seq <>)(lib.metadata/bulk-metadata-or-throw query :metadata/table <>))))@@ -454,8 +457,7 @@"Merge column metadata from the non-sandboxed version of the query into the sandboxed results `metadata`. This way thefinal results metadata coming back matches what we'd get if the query was not running in a sandbox."[original-metadata :- [:sequential ::mbql.s/legacy-column-metadata]- metadata :- [:map- [:cols [:sequential ::mbql.s/legacy-column-metadata]]]]+ metadata :- :metabase.query-processor.schema/metadata](letfn [(merge-cols [cols](let [col-name->expected-col (m/index-by :name original-metadata)](for [col cols]diff --git a/enterprise/backend/src/metabase_enterprise/sandbox/schema.clj b/enterprise/backend/src/metabase_enterprise/sandbox/schema.cljindex 967e19dc20c..cd2600f8c3e 100644--- a/enterprise/backend/src/metabase_enterprise/sandbox/schema.clj+++ b/enterprise/backend/src/metabase_enterprise/sandbox/schema.clj@@ -6,7 +6,8 @@[metabase.models.interface :as mi][metabase.util :as u][metabase.util.malli.registry :as mr]- [metabase.util.malli.schema :as ms]))+ [metabase.util.malli.schema :as ms]+ [metabase.warehouse-schema.schema]))(mr/def ::attribute-remappings"value must be a valid attribute remappings map (attribute name -> remapped name)"@@ -34,16 +35,15 @@(mr/def ::sandbox.attribute-remappings"The `:attribute_remappings` column of a Sandbox, decoded."- :map)+ ::attribute-remappings)(mr/def ::sandbox- "A Sandbox as selected from the app DB: every column of `:sandboxes`."- [:map {:closed true}- [:id ms/PositiveInt]- [:group_id ms/PositiveInt]- [:table_id ::lib.schema.id/table]- [:card_id [:maybe ::lib.schema.id/card]]- [:attribute_remappings [:maybe ::sandbox.attribute-remappings]]])+ "A Sandbox as selected from the app DB: every column of `:sandboxes`, plus `:table` some callers hydrate onto it."+ [:merge+ ::sandbox.update+ [:map {:closed true}+ [:id ms/PositiveInt]+ [:table {:optional true} [:maybe :metabase.warehouse-schema.schema/table]]]])(mr/def ::sandbox.update"What an update (or insert) of a Sandbox accepts: every column of `:sandboxes` except `id`, all optional."diff --git a/enterprise/backend/src/metabase_enterprise/scim/v2/api.clj b/enterprise/backend/src/metabase_enterprise/scim/v2/api.cljindex 47808e914e5..fba7ff0b575 100644--- a/enterprise/backend/src/metabase_enterprise/scim/v2/api.clj+++ b/enterprise/backend/src/metabase_enterprise/scim/v2/api.clj@@ -11,6 +11,7 @@[metabase.api.macros :as api.macros][metabase.models.interface :as mi][metabase.permissions.core :as perms]+ [metabase.permissions.schema :as permissions.schema][metabase.users.schema :as users.schema][metabase.util :as u][metabase.util.i18n :as i18n]@@ -166,7 +167,7 @@(mu/defn ^:private mb-user->scim :- SCIMUser"Given a Metabase user, returns a SCIM user."- [user]+ [user :- ::users.schema/user]{:schemas [user-schema-uri]:id (:entity_id user):userName (:email user)@@ -187,7 +188,7 @@(mu/defn ^:private scim-user->mb :- users.schema/NewUser"Given a SCIM user, returns a Metabase user."- [user]+ [user :- SCIMUser](let [{email :userName name-obj :name locale :locale is-active? :active} user{:keys [givenName familyName]} name-obj](merge@@ -202,7 +203,7 @@(mu/defn ^:private get-user-by-entity-id"Fetches a user by entity ID, or throws a 404"- [entity-id]+ [entity-id :- ms/NonBlankString](or (scim.db/scim-user-by-entity-id entity-id)(throw-scim-error 404 "User not found")))@@ -379,13 +380,13 @@(mu/defn ^:private get-group-by-entity-id"Fetches a group by entity ID, or throws a 404. Cannot fetch the Administrators or All Users groups, as these arestatic, nor data-app groups, as Metabase manages their membership itself, so none can be managed via SCIM."- [entity-id]+ [entity-id :- ms/NonBlankString](or (scim.db/scim-group-by-entity-id entity-id (hidden-group-ids))(throw-scim-error 404 "Group not found")))(mu/defn ^:private mb-group->scim :- SCIMGroup"Given a Metabase permissions group, returns a SCIM group."- [group]+ [group :- ::permissions.schema/permissions-group]{:schemas [group-schema-uri]:id (:entity_id group):members (mapdiff --git a/enterprise/backend/src/metabase_enterprise/security_center/matching.clj b/enterprise/backend/src/metabase_enterprise/security_center/matching.cljindex d794a6d0fe0..4e183847e60 100644--- a/enterprise/backend/src/metabase_enterprise/security_center/matching.clj+++ b/enterprise/backend/src/metabase_enterprise/security_center/matching.clj@@ -84,7 +84,7 @@"Execute a matching query against the appdb.Returns true if rows matched, false if no rows, :error if query failed.nil matching_query means 'affects all instances' — returns true."- [matching-query :- [:maybe :map]]+ [matching-query :- ::schema/matching-query](if (nil? matching-query)true(if-let [query (select-query-for-dialect matching-query)]diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/db.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/db.cljindex 0cc746fa705..724ff283919 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/db.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/db.clj@@ -7,6 +7,7 @@[metabase-enterprise.semantic-search.schema :as semantic-search.schema][metabase.app-db.core :as mdb][metabase.lib.schema.id :as lib.schema.id]+ [metabase.search.config :as search.config][metabase.search.scoring :as search.scoring][metabase.util.malli :as mu][metabase.util.malli.schema :as ms]@@ -94,10 +95,11 @@[search-results :- [:sequential [:map {:closed true}[:id [:or :string ms/PositiveInt]][:model :string]]]- search-ctx :- [:map+ search-ctx :- [:map {:closed true}[:current-user-id {:optional true} [:maybe ms/PositiveInt]]- [:context {:optional true} [:maybe :keyword]]- [:weights {:optional true} [:maybe [:map-of :keyword number?]]]]]+ [:context {:optional true} [:maybe :keyword]]+ [:weights {:optional true} [:maybe ::search.config/weights]]+ [:limit-int {:optional true} [:maybe ms/Int]]]](let [scorers (appdb-scoring/appdb-scorers search-ctx)](t2/query (cond-> (search.scoring/with-scores search-ctx scorers (search-index-select search-results))(:bookmarked scorers) (search.scoring/join-bookmarks (:current-user-id search-ctx))))))diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/embedding.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/embedding.cljindex a86c6400e00..3633909847c 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/embedding.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/embedding.clj@@ -476,7 +476,7 @@[{:keys [provider endpoint api-key model-name vector-dimensions texts record-tokens? extra-body snowplow?network-policy-floor instance-token?]:as opts}- :- [:map+ :- [:map {:closed true}[:provider :string][:endpoint :string][:api-key {:optional true} [:maybe :string]]@@ -485,9 +485,11 @@[:texts [:sequential :string]][:record-tokens? :boolean][:snowplow? {:optional true} [:maybe :boolean]]- [:extra-body {:optional true} [:maybe :map]]+ [:extra-body {:optional true} [:maybe [:map {:closed true}+ [:dimensions {:optional true} pos-int?]]]][:network-policy-floor {:optional true} [:maybe [:enum :external-only :allow-private :allow-all]]]- [:instance-token? {:optional true} [:maybe :boolean]]]]+ [:instance-token? {:optional true} [:maybe :boolean]]+ [:type {:optional true} [:maybe [:enum :query :index]]]]];; Outside the try: a malformed endpoint is neither a service failure nor something to log per batch.(let [policy-opts (llm.settings/llm-request-opts network-policy-floor endpoint)](trydiff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/schema.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/schema.cljindex 73ddc07f954..f995f7dfe1f 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/schema.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/schema.clj@@ -6,12 +6,10 @@(mr/def ::semantic-search-token-tracking"A SemanticSearchTokenTracking as selected from the app DB: every column of `:semantic_search_token_tracking`."- [:map {:closed true}- [:id ms/PositiveInt]- [:model_name :string]- [:request_type [:maybe [:or :keyword :string]]]- [:created_at ms/TemporalInstant]- [:total_tokens :int]])+ [:merge+ ::semantic-search-token-tracking.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::semantic-search-token-tracking.update"What an update (or insert) of a SemanticSearchTokenTracking accepts: every column of `:semantic_search_token_tracking` except `id`, all optional."diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/scoring.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/scoring.cljindex 32d81b796c5..777a2641836 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/scoring.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/scoring.clj@@ -201,7 +201,7 @@(seq appdb-scorers))search-results(->> (semantic-search.db/appdb-scored-rows (mapv #(select-keys % [:id :model]) search-results-to-score)- search-ctx)+ (select-keys search-ctx [:current-user-id :context :weights :limit-int]))(update-with-appdb-scores weights (keys appdb-scorers) search-results)(sort-by :score >)vec))))diff --git a/enterprise/backend/src/metabase_enterprise/serialization/cmd.clj b/enterprise/backend/src/metabase_enterprise/serialization/cmd.cljindex 255e2490433..4ebd8d9f284 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/cmd.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/cmd.clj@@ -32,14 +32,17 @@`opts` are passed to [[v2.load/load-metabase]]."[path :- :string- opts :- [:map+ opts :- [:map {:closed true}[:continue-on-error {:optional true} [:maybe :boolean]][:reindex? {:optional true} [:maybe :boolean]]];; Deliberately separate from the opts so it can't be set from the CLI.& {:keys [token-check?require-initialized-db?]:or {token-check? true- require-initialized-db? true}}]+ require-initialized-db? true}}+ :- [:maybe [:map {:closed true}+ [:token-check? {:optional true} [:maybe :boolean]]+ [:require-initialized-db? {:optional true} [:maybe :boolean]]]]](plugins/load-plugins!)(mdb/setup-db! :create-sample-content? false)(when (and require-initialized-db? (not (setup/has-user-setup)))@@ -59,7 +62,7 @@opts are passed to load-metabase"[path :- :string- opts :- [:map+ opts :- [:map {:closed true}[:continue-on-error {:optional true} [:maybe :boolean]][:full-stacktrace {:optional true} [:maybe :boolean]]]](let [timer (u/start-timer)diff --git a/enterprise/backend/src/metabase_enterprise/snippet_collections/models/native_query_snippet/permissions.clj b/enterprise/backend/src/metabase_enterprise/snippet_collections/models/native_query_snippet/permissions.cljindex bfe72945bee..6b5a55c38eb 100644--- a/enterprise/backend/src/metabase_enterprise/snippet_collections/models/native_query_snippet/permissions.clj+++ b/enterprise/backend/src/metabase_enterprise/snippet_collections/models/native_query_snippet/permissions.clj@@ -4,16 +4,20 @@[metabase-enterprise.snippet-collections.db :as snippet-collections.db][metabase.models.interface :as mi][metabase.native-query-snippets.core :as snippets]+ [metabase.native-query-snippets.schema :as snippets.schema][metabase.permissions.core :as perms][metabase.premium-features.core :refer [defenterprise]][metabase.remote-sync.core :as remote-sync]- [metabase.util.malli :as mu]- [metabase.util.malli.schema :as ms]))+ [metabase.util.malli :as mu]))(mu/defn- has-parent-collection-perms?- [snippet :- [:map [:collection_id [:maybe ms/PositiveInt]]]+ "Whether the current user has `read-or-write` permissions on `snippet`'s parent collection. `snippet` must not be+ nil: a nonexistent Snippet must fail loudly here rather than silently fall back to root-collection permissions."+ [snippet :- [:or+ ::snippets.schema/native-query-snippet+ ::snippets.schema/native-query-snippet.update]read-or-write :- [:enum :read :write]]- (mi/current-user-has-full-permissions? (perms/perms-objects-set-for-parent-collection "snippets" snippet read-or-write)))+ (mi/current-user-has-full-permissions? (perms/perms-objects-set-for-parent-collection "snippets" (:collection_id snippet) read-or-write)))(defenterprise can-read?"Can the current User read this `snippet`?"diff --git a/enterprise/backend/src/metabase_enterprise/sso/api/sso.clj b/enterprise/backend/src/metabase_enterprise/sso/api/sso.cljindex a9331e77b0a..74e48ddda8b 100644--- a/enterprise/backend/src/metabase_enterprise/sso/api/sso.clj+++ b/enterprise/backend/src/metabase_enterprise/sso/api/sso.clj@@ -51,7 +51,7 @@(throw e))))(mu/defn- sso-error-page- [^Throwable e log-direction :- [:enum :in :out]]+ [^Throwable e :- (ms/InstanceOfClass Throwable) log-direction :- [:enum :in :out]]{:status (get (ex-data e) :status-code 500):headers {"Content-Type" "text/html"}:body (stencil/render-file "metabase_enterprise/sandbox/api/error_page"diff --git a/enterprise/backend/src/metabase_enterprise/sso/schema.clj b/enterprise/backend/src/metabase_enterprise/sso/schema.cljindex c738e35f971..9f284554391 100644--- a/enterprise/backend/src/metabase_enterprise/sso/schema.clj+++ b/enterprise/backend/src/metabase_enterprise/sso/schema.clj@@ -6,13 +6,10 @@(mr/def ::sso-relay-state"A SsoRelayState as selected from the app DB: every column of `:sso_relay_state`."- [:map {:closed true}- [:id :string]- [:continue_url :string]- [:origin [:maybe :string]]- [:embedding :boolean]- [:expires_at ms/TemporalInstant]- [:created_at ms/TemporalInstant]])+ [:merge+ ::sso-relay-state.update+ [:map {:closed true}+ [:id :string]]])(mr/def ::sso-relay-state.update"What an update (or insert) of a SsoRelayState accepts: every column of `:sso_relay_state` except `id`, all optional."diff --git a/enterprise/backend/src/metabase_enterprise/stale/api.clj b/enterprise/backend/src/metabase_enterprise/stale/api.cljindex 1c144d43671..6d630b46dc1 100644--- a/enterprise/backend/src/metabase_enterprise/stale/api.clj+++ b/enterprise/backend/src/metabase_enterprise/stale/api.clj@@ -58,11 +58,11 @@(defmethod present-model-items :model/Card [_ cards](->> (t2/hydrate (stale.db/stale-cards (set (map :id cards))) :can_write :can_delete :can_restore [:collection :effective_location] :dashboard_count [:dashboard :moderation_status])+ (map #(assoc % :fully_parameterized (queries/fully-parameterized? %)))present-collections(map (fn [card](-> card(assoc :model (if (queries/model? card) "dataset" "card"))- (assoc :fully_parameterized (queries/fully-parameterized? card))(dissoc :dataset_query))))))(defn- annotate-dashboard-with-collection-infodiff --git a/enterprise/backend/src/metabase_enterprise/stale/db.clj b/enterprise/backend/src/metabase_enterprise/stale/db.cljindex 51a913c2ea9..dd9a412384e 100644--- a/enterprise/backend/src/metabase_enterprise/stale/db.clj+++ b/enterprise/backend/src/metabase_enterprise/stale/db.clj@@ -3,6 +3,7 @@additional logic, so the rest of the module only touches `toucan2.core` for hydration."(:require[metabase.lib.schema.id :as lib.schema.id]+ [metabase.staleness.core :as staleness][metabase.util.malli :as mu][toucan2.core :as t2]))@@ -77,7 +78,7 @@"A page of `:id`/`:model` rows from the union of `union-queries`, sorted by `sort-column` (`:name` or`:last_used_at`) in `sort-direction`, skipping `offset` and returning up to `limit` (either may be nil for norestriction)."- [union-queries :- [:sequential :map]+ [union-queries :- [:sequential ::staleness/query]sort-column :- [:enum :name :last_used_at]sort-direction :- [:enum :asc :desc]limit :- [:maybe :int]@@ -93,6 +94,6 @@(mu/defn stale-content-count"The total count of rows across every page [[stale-content-rows]] would return for `union-queries`."- [union-queries :- [:sequential :map]]+ [union-queries :- [:sequential ::staleness/query]](:count (t2/query-one {:select [[:%count.* :count]]:from (stale-content-union union-queries)})))diff --git a/enterprise/backend/src/metabase_enterprise/stale/impl.clj b/enterprise/backend/src/metabase_enterprise/stale/impl.cljindex e0d4647fabc..2e2ca167274 100644--- a/enterprise/backend/src/metabase_enterprise/stale/impl.clj+++ b/enterprise/backend/src/metabase_enterprise/stale/impl.clj@@ -8,7 +8,7 @@(set! *warn-on-reflection* true)(def ^:private FindStaleContentArgs- [:map+ [:map {:closed true}[:collection-ids [:set {:doc "The set of collection IDs to search for stale content."} [:maybe :int]]][:cutoff-date [:time/local-date {:doc "The cutoff date for stale content."}]][:limit [:maybe {:doc "The limit for pagination."} :int]]diff --git a/enterprise/backend/src/metabase_enterprise/support_access_grants/db.clj b/enterprise/backend/src/metabase_enterprise/support_access_grants/db.cljindex 6515780e75f..f05ef8f1466 100644--- a/enterprise/backend/src/metabase_enterprise/support_access_grants/db.clj+++ b/enterprise/backend/src/metabase_enterprise/support_access_grants/db.clj@@ -146,8 +146,20 @@[:confirmed_at {:optional true} [:maybe ms/TemporalInstant]][:created_at {:optional true} ms/TemporalInstant][:updated_at {:optional true} ms/TemporalInstant]- [:credentials {:optional true} [:maybe :map]]- [:metadata {:optional true} [:maybe :map]]]]+ [:credentials {:optional true}+ [:maybe [:map {:closed true}+ [:token_hash :string]+ [:expires_at ms/TemporalInstant]+ [:grant_ends_at ms/TemporalInstant]+ [:consumed_at [:maybe ms/TemporalInstant]]]]]+ [:metadata {:optional true}+ [:maybe [:map {:closed true}+ [:email ms/Email]+ [:ip_address {:optional true} [:maybe :string]]+ [:request_context {:optional true}+ [:map {:closed true}+ [:user_agent {:optional true} [:maybe :string]]+ [:timestamp {:optional true} [:or ms/TemporalInstant :string]]]]]]]]](t2/update! :model/AuthIdentity auth-identity-id changes))(mu/defn expire-auth-identities!diff --git a/enterprise/backend/src/metabase_enterprise/support_access_grants/provider.clj b/enterprise/backend/src/metabase_enterprise/support_access_grants/provider.cljindex 2e62fb29769..9a37a94769d 100644--- a/enterprise/backend/src/metabase_enterprise/support_access_grants/provider.clj+++ b/enterprise/backend/src/metabase_enterprise/support_access_grants/provider.clj@@ -39,7 +39,7 @@[token :- :stringgrant-ends-at :- ::schema/timestamp& {:keys [expires-in-ms]- :or {expires-in-ms (* 48 60 60 1000)}}]+ :or {expires-in-ms (* 48 60 60 1000)}} :- [:maybe [:map {:closed true} [:expires-in-ms {:optional true} [:maybe :int]]]]]{:token_hash (u.password/hash-bcrypt token):expires_at (t/plus (t/instant) (t/millis expires-in-ms)):grant_ends_at grant-ends-at@@ -62,7 +62,7 @@Returns the plaintext token string that should be sent to the user via email."[user-id :- ms/PositiveInt- grant :- [:maybe [:map [:grant_end_timestamp ::schema/timestamp]]]]+ grant :- [:maybe [:or ::schema/support-access-grant-log ::schema/grant-response]]](when-not grant(throw (ex-info "Cannot create support access reset token: no grant provided"{:status-code 400})))diff --git a/enterprise/backend/src/metabase_enterprise/support_access_grants/schema.clj b/enterprise/backend/src/metabase_enterprise/support_access_grants/schema.cljindex 574084b4dfb..bb6098c98ae 100644--- a/enterprise/backend/src/metabase_enterprise/support_access_grants/schema.clj+++ b/enterprise/backend/src/metabase_enterprise/support_access_grants/schema.clj@@ -28,7 +28,7 @@(mr/def ::grant-response"Schema for a support access grant object in API responses."- [:map+ [:map {:closed true}[:id ms/PositiveInt][:user_id ms/PositiveInt][:user_name [:maybe :string]]@@ -57,17 +57,10 @@(mr/def ::support-access-grant-log"A SupportAccessGrantLog as selected from the app DB: every column of `:support_access_grant_log`."- [:map {:closed true}- [:id ms/PositiveInt]- [:user_id [:maybe ::lib.schema.id/user]]- [:ticket_number [:maybe :string]]- [:notes [:maybe :string]]- [:grant_start_timestamp ms/TemporalInstant]- [:grant_end_timestamp ms/TemporalInstant]- [:revoked_at [:maybe ms/TemporalInstant]]- [:revoked_by_user_id [:maybe ::lib.schema.id/user]]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]])+ [:merge+ ::support-access-grant-log.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::support-access-grant-log.update"What an update (or insert) of a SupportAccessGrantLog accepts: every column of `:support_access_grant_log` except `id`, all optional."diff --git a/enterprise/backend/src/metabase_enterprise/tenants/schema.clj b/enterprise/backend/src/metabase_enterprise/tenants/schema.cljindex 7552aeb6682..535df16eeca 100644--- a/enterprise/backend/src/metabase_enterprise/tenants/schema.clj+++ b/enterprise/backend/src/metabase_enterprise/tenants/schema.clj@@ -11,15 +11,10 @@(mr/def ::tenant"A Tenant as selected from the app DB: every column of `:tenant`."- [:map {:closed true}- [:id ms/PositiveInt]- [:name :string]- [:slug :string]- [:is_active :boolean]- [:updated_at ms/TemporalInstant]- [:created_at ms/TemporalInstant]- [:attributes [:maybe ::tenant.attributes]]- [:tenant_collection_id ::lib.schema.id/collection]])+ [:merge+ ::tenant.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::tenant.update"What an update (or insert) of a Tenant accepts: every column of `:tenant` except `id`, all optional."diff --git a/enterprise/backend/src/metabase_enterprise/transforms_inspector/context.clj b/enterprise/backend/src/metabase_enterprise/transforms_inspector/context.cljindex c1003c55f0f..cb1706ce1c5 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_inspector/context.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_inspector/context.clj@@ -1,17 +1,22 @@(ns metabase-enterprise.transforms-inspector.context"Context building for Transform Inspector."(:require- [clojure.set :as set][clojure.string :as str][metabase-enterprise.transforms-inspector.db :as transforms-inspector.db][metabase-enterprise.transforms-inspector.query-analysis :as query-analysis][metabase-enterprise.transforms-inspector.schema :as transforms-inspector.schema][metabase.driver :as driver][metabase.lib.core :as lib]+ [metabase.lib.schema :as lib.schema]+ [metabase.lib.schema.id :as lib.schema.id]+ [metabase.lib.schema.join :as lib.schema.join]+ [metabase.lib.schema.metadata.fingerprint :as lib.schema.metadata.fingerprint][metabase.query-processor.preprocess :as qp.preprocess][metabase.transforms-base.interface :as transforms-base.i][metabase.transforms-base.util :as transforms-base.u]+ [metabase.transforms.schema :as transforms.schema][metabase.util :as u]+ [metabase.util.honey-sql-2 :as h2x][metabase.util.log :as log][metabase.util.malli :as mu][metabase.util.malli.registry :as mr]))@@ -91,32 +96,39 @@;;; -------------------------------------------------- Field Metadata --------------------------------------------------(mu/defn- get-field-stats :- [:maybe ::transforms-inspector.schema/field-stats]- "Extract fingerprint stats for a field."- [field]- (let [fp (:fingerprint field)]- (not-empty- (into {}- (remove (comp nil? val))- (cond-> (merge (when-let [dc (get-in fp [:global :distinct-count])]- {:distinct_count dc})- (select-keys (get-in fp [:type :type/Number]) [:min :max :avg :q1 :q3])- (select-keys (get-in fp [:type :type/DateTime]) [:earliest :latest]))- (some? (get-in fp [:global :nil%]))- (assoc :nil_percent (get-in fp [:global :nil%])))))))+ "Extract fingerprint stats for a field's fingerprint."+ [fingerprint :- [:maybe ::lib.schema.metadata.fingerprint/fingerprint]]+ (not-empty+ (into {}+ (remove (comp nil? val))+ (cond-> (merge (when-let [dc (get-in fingerprint [:global :distinct-count])]+ {:distinct_count dc})+ (select-keys (get-in fingerprint [:type :type/Number]) [:min :max :avg :q1 :q3])+ (select-keys (get-in fingerprint [:type :type/DateTime]) [:earliest :latest]))+ (some? (get-in fingerprint [:global :nil%]))+ (assoc :nil_percent (get-in fingerprint [:global :nil%]))))))(mu/defn- collect-field-metadata :- [:sequential ::transforms-inspector.schema/field]"Collect metadata for fields in a table."- [table-id]+ [table-id :- ::lib.schema.id/table](let [fields (transforms-inspector.db/active-fields-for-table table-id)](mapv (fn [field]- (cond-> (select-keys field [:id :name :display_name :base_type :semantic_type])- (get-field-stats field)- (assoc :stats (get-field-stats field))))+ (let [stats (get-field-stats (:fingerprint field))]+ (cond-> (select-keys field [:id :name :display_name :base_type :semantic_type])+ stats (assoc :stats stats))))fields)))+(def ^:private TableRef+ "A table reference: table, name, schema, and owning database, keyed for lookups."+ [:map {:closed true}+ [:table-id ::lib.schema.id/table]+ [:table-name :string]+ [:schema [:maybe :string]]+ [:db-id ::lib.schema.id/database]])+(mu/defn- build-table-info :- ::transforms-inspector.schema/table"Build table info map with fields."- [{:keys [table-id table-name schema db-id]}]+ [{:keys [table-id table-name schema db-id]} :- TableRef](let [fields (collect-field-metadata table-id)]{:table_id table-id:table_name table-name@@ -195,11 +207,48 @@:name (:name returned-col):id (:id returned-col)}]})))+(mr/def ::honeysql-value+ "A HoneySQL scalar value from a parsed native join clause: an identifier or a lifted SQL literal."+ [:or h2x/Identifier [:tuple [:= :lift] [:maybe [:or :string number? :boolean]]]])++(mr/def ::honeysql-condition+ "A single HoneySQL boolean condition from a parsed native join clause's ON expression."+ [:or+ [:tuple :keyword [:ref ::honeysql-condition] [:ref ::honeysql-condition]]+ [:tuple :keyword ::honeysql-value ::honeysql-value]])++(mr/def ::honeysql-table-ref+ "A HoneySQL FROM/JOIN table reference: an identifier, optionally aliased."+ [:or [:tuple h2x/Identifier] [:tuple h2x/Identifier :keyword]])++(mr/def ::join-structure-entry+ [:map {:closed true}+ [:strategy {:optional true} [:maybe [:or :keyword :string]]]+ [:alias {:optional true} [:maybe :string]]+ [:source-table {:optional true} [:maybe ::lib.schema.id/table]]+ [:conditions {:optional true} ::lib.schema.join/conditions]+ [:join-table {:optional true} ::honeysql-table-ref]+ [:join-condition {:optional true} [:maybe [:or ::honeysql-condition+ [:and+ [:fn {:error/message "an :and of conditions"} #(= :and (first %))]+ [:sequential [:or [:= :and] [:ref ::honeysql-condition]]]]]]]])++(def ^:private QueryInfo+ [:map {:closed true}+ [:preprocessed-query {:optional true} [:maybe ::lib.schema/query]]+ [:driver {:optional true} [:maybe :keyword]]+ [:from-table-id {:optional true} [:maybe ::lib.schema.id/table]]+ [:from-table {:optional true} [:maybe ::honeysql-table-ref]]+ [:join-structure {:optional true} [:maybe [:sequential ::join-structure-entry]]]+ [:visited-fields {:optional true} [:maybe [:map {:closed true} [:all {:optional true} [:maybe [:set ::lib.schema.id/field]]]]]]])+(mu/defn- match-columns :- [:maybe [:sequential ::column-match]]"Find columns that relate between input and output tables.Uses field ID-based matching for MBQL queries (more accurate),falls back to name-based matching for native queries."- [sources target {:keys [preprocessed-query join-structure]}]+ [sources :- [:sequential ::transforms-inspector.schema/table]+ target :- ::transforms-inspector.schema/table+ {:keys [preprocessed-query join-structure]} :- [:maybe QueryInfo]](if preprocessed-query(match-columns-mbql preprocessed-query sources target)(match-columns-by-name sources target join-structure)))@@ -253,12 +302,15 @@(mu/defn build-context :- ::context"Build context for lens discovery and generation."- [transform]+ [transform :- ::transforms.schema/transform](let [source-type (transforms-base.u/transform-source-type (:source transform))sources-info (mapv build-table-info (extract-sources transform))target-table (get-target-table transform)target-info (when target-table- (build-table-info (set/rename-keys target-table {:id :table-id :name :table-name :db_id :db-id})))+ (build-table-info {:table-id (:id target-table)+ :table-name (:name target-table)+ :schema (:schema target-table)+ :db-id (:db_id target-table)}))query-info (query-analysis/analyze-query transform source-type sources-info)join-structure (:join-structure query-info)column-matches (when (and (seq sources-info) target-info)diff --git a/enterprise/backend/src/metabase_enterprise/transforms_inspector/core.clj b/enterprise/backend/src/metabase_enterprise/transforms_inspector/core.cljindex 2bb2e122d2f..c07bf5d9f74 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_inspector/core.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_inspector/core.clj@@ -8,7 +8,8 @@[metabase-enterprise.transforms-inspector.lens.generic][metabase-enterprise.transforms-inspector.lens.join-analysis][metabase-enterprise.transforms-inspector.lens.unmatched-rows]- [metabase-enterprise.transforms-inspector.schema]+ [metabase-enterprise.transforms-inspector.schema :as inspector.schema]+ [metabase.transforms.schema :as transforms.schema][metabase.util.i18n :refer [tru]][metabase.util.malli :as mu]))@@ -22,7 +23,7 @@"Discover available lenses for a transform.Returns structural metadata and available lens types.This is a cheap operation - no query execution."- [transform :- :map]+ [transform :- ::transforms.schema/transform](let [{:keys [sources target] :as ctx} (context/build-context transform)](if-not target{:name (tru "Transform Inspector: {0}" (:name transform))@@ -43,8 +44,8 @@"Get full lens contents for a transform.Returns sections, cards, and trigger definitions.Optional params can filter/customize drill lens output."- [transform :- :map+ [transform :- ::transforms.schema/transformlens-id :- :string- params :- [:maybe :map]]+ params :- [:maybe ::inspector.schema/lens-params.request]](let [ctx (context/build-context transform)](lens.core/get-lens ctx lens-id params)))diff --git a/enterprise/backend/src/metabase_enterprise/transforms_inspector/schema.clj b/enterprise/backend/src/metabase_enterprise/transforms_inspector/schema.cljindex b7631c76aa3..5b502899883 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_inspector/schema.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_inspector/schema.clj@@ -15,7 +15,7 @@(mr/def ::field-stats"Statistics for a field from fingerprints."- [:map+ [:map {:closed true}[:distinct_count {:optional true} :int][:nil_percent {:optional true} :double];; Number stats@@ -30,7 +30,7 @@(mr/def ::field"Field metadata for inspector."- [:map+ [:map {:closed true}[:id pos-int?][:name :string][:display_name {:optional true} [:maybe :string]]@@ -40,7 +40,7 @@(mr/def ::table"Table metadata."- [:map+ [:map {:closed true}[:table_id pos-int?][:table_name :string][:schema {:optional true} [:maybe :string]]diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/base.cljindex c92b318db34..ded2d26a271 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj@@ -261,7 +261,13 @@Options:- `with-stage-timing-fn` - optional, (fn [run-id stage thunk] result) for instrumentation"- [{:keys [source] :as transform} db run-id cancel-chan message-log {:keys [with-stage-timing-fn source-range-params]}]+ [{:keys [source] :as transform} :- ::transforms-base.schema/transform+ db :- :metabase.warehouses.schema/database+ run-id :- pos-int?+ cancel-chan :- ::transforms-base.schema/chan+ message-log :- ::transforms-base.schema/atom+ {:keys [with-stage-timing-fn source-range-params]}+ :- ::transforms-base.schema/execute-base-options];; Resolve name-based source table refs to table IDs (throws if any not found)(let [resolved-source-tables (transforms-base.u/resolve-source-tables (:source-tables source))](with-open [shared-storage-ref (s3/open-shared-storage! resolved-source-tables)]diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/models/python_library.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/models/python_library.cljindex 712d50ba319..b1f070528e0 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/models/python_library.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/models/python_library.clj@@ -1,6 +1,7 @@(ns metabase-enterprise.transforms-python.models.python-library(:require[metabase-enterprise.transforms-python.db :as transforms-python.db]+ [metabase-enterprise.transforms-python.schema][metabase.api.common :as api][metabase.events.core :as events][metabase.models.interface :as mi]diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/schema.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/schema.cljindex 925fb8f0d92..3c04089e910 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/schema.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/schema.clj@@ -6,13 +6,10 @@(mr/def ::python-library"A PythonLibrary as selected from the app DB: every column of `:python_library`."- [:map {:closed true}- [:id ms/PositiveInt]- [:path :string]- [:source [:maybe [:or :keyword :string]]]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]- [:entity_id :string]])+ [:merge+ ::python-library.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::python-library.update"What an update (or insert) of a PythonLibrary accepts: every column of `:python_library` except `id`, all optional."diff --git a/enterprise/backend/src/metabase_enterprise/upload_management/db.clj b/enterprise/backend/src/metabase_enterprise/upload_management/db.cljindex e4f123e5a1c..6666afe3347 100644--- a/enterprise/backend/src/metabase_enterprise/upload_management/db.clj+++ b/enterprise/backend/src/metabase_enterprise/upload_management/db.clj@@ -25,5 +25,5 @@(mu/defn table"The Table with `table-id`, or nil."- [table-id :- ::lib.schema.id/table]+ [table-id :- [:maybe ::lib.schema.id/table]](t2/select-one :model/Table :id table-id {:from [(warehouse-schema-overlay/table-query)]}))diff --git a/enterprise/backend/test/metabase/notification/payload/core_ee_test.clj b/enterprise/backend/test/metabase/notification/payload/core_ee_test.cljindex 8d7075d8de4..8d43cb9a616 100644--- a/enterprise/backend/test/metabase/notification/payload/core_ee_test.clj+++ b/enterprise/backend/test/metabase/notification/payload/core_ee_test.clj@@ -18,7 +18,7 @@site-url "https://metabase.com"admin-email "[email protected]"](is (= {:payload_type :notification/system-event- :payload {:event_info {:foo :bar}+ :payload {:event_info {}:event_topic :event/user-joined:custom {}}:context {:application_name "Metabase Test"@@ -32,5 +32,5 @@:creator nil}(-> (notification.payload/notification-payload {:payload_type :notification/system-event:payload {:event_topic :event/user-joined- :event_info {:foo :bar}}})+ :event_info {}}})(update-in [:context :style :button] string?))))))))diff --git a/enterprise/backend/test/metabase_enterprise/advanced_permissions/common_test.clj b/enterprise/backend/test/metabase_enterprise/advanced_permissions/common_test.cljindex d3a196c07e7..6713c6d5985 100644--- a/enterprise/backend/test/metabase_enterprise/advanced_permissions/common_test.clj+++ b/enterprise/backend/test/metabase_enterprise/advanced_permissions/common_test.clj@@ -1048,7 +1048,7 @@[{(:id table-b) :query-builder} false "Data permissions only on another table in the same schema should fail"]]](testing test-string(mt/with-all-users-data-perms-graph! {db-id {:view-data :unrestricted- :create-queries {schema-name schema-perms}}}+ :create-queries {(or schema-name "") schema-perms}}}(if can-append?(is (some? (append-csv!)))(is (thrown-with-msg?diff --git a/enterprise/backend/test/metabase_enterprise/data_apps/query_definition_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/query_definition_test.cljindex b42527c5cd5..58c3ee1b716 100644--- a/enterprise/backend/test/metabase_enterprise/data_apps/query_definition_test.clj+++ b/enterprise/backend/test/metabase_enterprise/data_apps/query_definition_test.clj@@ -38,9 +38,10 @@(is (not (mr/validate ::query-definition/query-definition {:stages []})))(is (not (mr/validate ::query-definition/query-definition {:stages [{}]}))))-(deftest ^:parallel internal-column-schemas-remain-open-test+(deftest ^:parallel column-unit-only-allowed-with-bucketing-test(let [column {:type :column :name "DATE" :unit :month}]- (is (mr/validate ::test-spec/test-column-spec column))+ (is (not (mr/validate ::test-spec/test-column-spec column)))+ (is (mr/validate ::test-spec/test-breakout-spec column))(is (not (mr/validate ::query-definition/column column)))(is (mr/validate ::query-definition/breakout column))))diff --git a/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.clj b/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.cljindex a4dcb7e8994..77a30cfc7bc 100644--- a/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.clj+++ b/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.clj@@ -1165,9 +1165,34 @@[published?](with-meta{:library {:score 0 :components {}} :universe {:score 0 :components {}}- :metabot {:score 0 :components {}} :meta {}}+ :metabot {:score 0 :components {}}+ :meta {:formula-version 1 :format-version 1 :synonym-threshold 0.8 :weights complexity/weights}}{:metabase-enterprise.data-complexity-score.complexity/snowplow-published? published?}))+(def ^:private zero-leaf+ "A minimal, schema-valid `::data-complexity-score.leaf`."+ {:measurement 0.0 :score 0})++(def ^:private zero-catalog+ "A minimal, schema-valid `::data-complexity-score.catalog` with all sub-scores zeroed out."+ {:score 0+ :components {:size {:score 0 :components {:entity-count zero-leaf :field-count zero-leaf}}+ :ambiguity {:score 0 :components {:name-collisions zero-leaf+ :synonym-pairs zero-leaf+ :repeated-measures zero-leaf}}}})++(defn- labeled-score+ "A full, valid `score-data` map carrying `label` in `:text-variant` so a test can tell rows apart."+ [label]+ {:library zero-catalog+ :universe zero-catalog+ :metabot zero-catalog+ :meta {:formula-version 1+ :format-version 1+ :synonym-threshold 0.8+ :weights complexity/weights+ :text-variant (keyword label)}})+(deftest ^:synchronized latest-score-filters-by-fingerprint-test(testing "the overview cache only returns scores matching the current scoring fingerprint"(mt/initialize-if-needed! :db)@@ -1175,11 +1200,11 @@fingerprint "latest-score-test/current"](try(t2/delete! :model/DataComplexityScore :fingerprint [:in [other-fingerprint fingerprint]])- (data-complexity-score/record-score! other-fingerprint "appdb" {:meta {:label "other"}})- (data-complexity-score/record-score! fingerprint "appdb" {:meta {:label "older"}})- (data-complexity-score/record-score! fingerprint "appdb" {:meta {:label "newer"}})+ (data-complexity-score/record-score! other-fingerprint "appdb" (labeled-score "other"))+ (data-complexity-score/record-score! fingerprint "appdb" (labeled-score "older"))+ (data-complexity-score/record-score! fingerprint "appdb" (labeled-score "newer"))(let [score (data-complexity-score/latest-score fingerprint)]- (is (= "newer" (get-in score [:meta :label])))+ (is (= :newer (get-in score [:meta :text-variant])))(is (some? (get-in score [:meta :calculated-at]))))(finally(t2/delete! :model/DataComplexityScore :fingerprint [:in [other-fingerprint fingerprint]]))))))@@ -1195,13 +1220,13 @@(let [fingerprint "latest-score-source-test/shared"](try(t2/delete! :model/DataComplexityScore :fingerprint fingerprint)- (data-complexity-score/record-score! fingerprint "appdb" {:meta {:label "appdb-row"}})- (data-complexity-score/record-score! fingerprint "representation:abcd" {:meta {:label "representation-row"}})- (is (= "appdb-row"- (get-in (data-complexity-score/latest-score fingerprint) [:meta :label]))+ (data-complexity-score/record-score! fingerprint "appdb" (labeled-score "appdb-row"))+ (data-complexity-score/record-score! fingerprint "representation:abcd" (labeled-score "representation-row"))+ (is (= :appdb-row+ (get-in (data-complexity-score/latest-score fingerprint) [:meta :text-variant]))"default source=\"appdb\" skips past the newer representation-tagged row")- (is (= "representation-row"- (get-in (data-complexity-score/latest-score fingerprint "representation:abcd") [:meta :label]))+ (is (= :representation-row+ (get-in (data-complexity-score/latest-score fingerprint "representation:abcd") [:meta :text-variant]))"an explicit representation source still resolves its own row")(finally(t2/delete! :model/DataComplexityScore :fingerprint fingerprint))))))@@ -1534,7 +1559,7 @@(let [score {:library {:score 0 :components {}}:universe {:score 0 :components {}}:metabot {:score 0 :components {}}- :meta {:formula-version 1 :synonym-threshold 0.8 :weights {}}}]+ :meta {:formula-version 1 :format-version 1 :synonym-threshold 0.8 :weights complexity/weights}}](testing "publish succeeds → tagged true"(snowplow-test/with-fake-snowplow-collector(is (true? (::complexity/snowplow-published? (meta (complexity/republish-score! score)))))))@@ -1555,8 +1580,9 @@:universe {:score 2 :components {}}:metabot {:score 3 :components {}}:meta {:formula-version 1+ :format-version 1:synonym-threshold 0.8- :weights {}+ :weights complexity/weights:text-variant :names-split:embedding-model {:provider :ai-service :model-name "minilm" :model-dimensions 384}}}](trydiff --git a/enterprise/backend/test/metabase_enterprise/dependencies/calculation_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/calculation_test.cljindex 27eff7f98e4..1d0e68a3f3a 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/calculation_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/calculation_test.clj@@ -163,8 +163,9 @@(mt/with-temp [:model/Transform transform {:name "Test Transform":source {:type :query:query query}- :target {:schema "PUBLIC"- :name "test_output"}}]+ :target {:type "table"+ :schema "PUBLIC"+ :name "test_output"}}](is (= {:card #{}:measure #{}:segment #{}@@ -186,8 +187,9 @@(mt/with-temp [:model/Transform transform {:name "Test Transform":source {:type :query:query query}- :target {:schema "PUBLIC"- :name "test_output"}}]+ :target {:type "table"+ :schema "PUBLIC"+ :name "test_output"}}](is (= {:card #{}:measure #{}:segment #{}@@ -208,8 +210,9 @@(mt/with-temp [:model/Transform transform {:name "Test Transform":source {:type :query:query query}- :target {:schema "PUBLIC"- :name "test_output"}}]+ :target {:type "table"+ :schema "PUBLIC"+ :name "test_output"}}](is (= {:card #{}:measure #{}:segment #{}@@ -226,8 +229,9 @@(transforms.tu/source-table-entry "ORDERS" orders-id)]:source-database (mt/id):body "..."}- :target {:schema "PUBLIC"- :name "test_output"}}]+ :target {:type "table"+ :schema "PUBLIC"+ :name "test_output"}}](is (= {:table #{products-id orders-id}}(calculation/calculate-deps :transform transform)))))))@@ -244,7 +248,8 @@(mt/with-temp [:model/Transform transform {:name "Table Tag Transform":source {:type :query:query query}- :target {:schema "PUBLIC"+ :target {:type "table"+ :schema "PUBLIC":name "test_output"}}](is (=? {:table #(contains? % products-id)} (calculation/calculate-deps :transform transform))))))))@@ -453,18 +458,18 @@(deftest ^:parallel upstream-deps-document-placeholder-ids-test(testing "nil/zero placeholder ids in smartLink and cardEmbed nodes are not collected as deps"- (let [document {:content_type "application/json+vnd.prose-mirror"- :document {:type "doc"- :content [{:type "paragraph"- :content [{:type "smartLink"- :attrs {"entityId" nil "model" "card"}}- {:type "smartLink"- :attrs {"entityId" 0 "model" "dashboard"}}- {:type "smartLink"- :attrs {"entityId" 17 "model" "card"}}]}- {:type "cardEmbed" :attrs {"id" nil}}- {:type "cardEmbed" :attrs {"id" 0}}- {:type "cardEmbed" :attrs {"id" 23}}]}}]+ (mt/with-temp [:model/Document document {:content_type "application/json+vnd.prose-mirror"+ :document {:type "doc"+ :content [{:type "paragraph"+ :content [{:type "smartLink"+ :attrs {"entityId" nil "model" "card"}}+ {:type "smartLink"+ :attrs {"entityId" 0 "model" "dashboard"}}+ {:type "smartLink"+ :attrs {"entityId" 17 "model" "card"}}]}+ {:type "cardEmbed" :attrs {"id" nil}}+ {:type "cardEmbed" :attrs {"id" 0}}+ {:type "cardEmbed" :attrs {"id" 23}}]}}](is (= {:card #{17 23}}(calculation/calculate-deps :document document))))))diff --git a/enterprise/backend/test/metabase_enterprise/internal_stats/metabot_test.clj b/enterprise/backend/test/metabase_enterprise/internal_stats/metabot_test.cljindex ddf9181a439..55e21c37d09 100644--- a/enterprise/backend/test/metabase_enterprise/internal_stats/metabot_test.clj+++ b/enterprise/backend/test/metabase_enterprise/internal_stats/metabot_test.clj@@ -127,9 +127,8 @@(is (every? false? (map :ai_proxied logs)))))(testing "usage keys are provider/model (metabase/ prefix stripped)";; accumulate-usage-xf strips metabase/ prefix → "anthropic/claude-sonnet-4-6"- ;; JSON roundtrip keywordizes → :anthropic/claude-sonnet-4-6(let [msg (t2/select-one :model/MetabotMessage :conversation_id conv-1 :role :assistant)]- (is (contains? (:usage msg) (keyword "anthropic" "claude-sonnet-4-6"))+ (is (contains? (:usage msg) "anthropic/claude-sonnet-4-6")"usage key should be provider/model without metabase/ prefix")));; -- Verify stats aggregation --diff --git a/enterprise/backend/test/metabase_enterprise/replacement/api_test.clj b/enterprise/backend/test/metabase_enterprise/replacement/api_test.cljindex e9ea4ca5050..3d1c9eeccfb 100644--- a/enterprise/backend/test/metabase_enterprise/replacement/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/replacement/api_test.clj@@ -129,7 +129,7 @@{:source {:type "query":query (lib/query mp (lib.metadata/card mp old-id))}:name "acceptance_transform"- :target {:database (mt/id) :table "acceptance_transform"}}+ :target {:database (mt/id) :type "table" :name "acceptance_transform"}}:model/Dashboard {dashboard-id :id}{:name "Acceptance Dashboard"}diff --git a/enterprise/backend/test/metabase_enterprise/replacement/field_refs_test.clj b/enterprise/backend/test/metabase_enterprise/replacement/field_refs_test.cljindex 74a6068282f..fd1751a22b1 100644--- a/enterprise/backend/test/metabase_enterprise/replacement/field_refs_test.clj+++ b/enterprise/backend/test/metabase_enterprise/replacement/field_refs_test.clj@@ -292,7 +292,7 @@(lib/filter (lib/= (lib.metadata/field mp (mt/id :orders :id)) 1)))](mt/with-temp [:model/Transform {transform-id :id} {:name "test transform":source {:type "query" :query query}- :target {:database (mt/id) :table "out"}}]+ :target {:database (mt/id) :type "table" :name "out"}}](replacement.field-refs/upgrade-field-refs! [:transform transform-id])(is (=? {:source {:type :query:query {:stages [{:source-table (mt/id :orders)@@ -306,7 +306,7 @@query (lib/native-query mp "SELECT * FROM orders")](mt/with-temp [:model/Transform {transform-id :id} {:name "test transform":source {:type "query" :query query}- :target {:database (mt/id) :table "out"}}]+ :target {:database (mt/id) :type "table" :name "out"}}](is (nil? (replacement.field-refs/upgrade-field-refs! [:transform transform-id])))))))(deftest transform-upgrade-field-refs!-broken-query-test@@ -314,7 +314,7 @@(let [mp (mt/metadata-provider)](mt/with-temp [:model/Transform {transform-id :id} {:name "test transform":source {:type "query" :query (lib/native-query mp "SELECT 1")}- :target {:database (mt/id) :table "out"}}]+ :target {:database (mt/id) :type "table" :name "out"}}];; simulate a broken query by updating source directly in the DB(t2/query-one {:update :transform:set {:source "{\"type\":\"query\",\"query\":{}}"}@@ -327,7 +327,7 @@query (lib/query mp (lib.metadata/table mp (mt/id :orders)))](mt/with-temp [:model/Transform {transform-id :id, updated-at :updated_at} {:name "test transform":source {:type "query" :query query}- :target {:database (mt/id) :table "out"}}]+ :target {:database (mt/id) :type "table" :name "out"}}](replacement.field-refs/upgrade-field-refs! [:transform transform-id])(is (= updated-at (:updated_at (t2/select-one :model/Transform transform-id))))))))diff --git a/enterprise/backend/test/metabase_enterprise/replacement/source_swap_test.clj b/enterprise/backend/test/metabase_enterprise/replacement/source_swap_test.cljindex 744ebeb1dc3..8a4c0ea27ba 100644--- a/enterprise/backend/test/metabase_enterprise/replacement/source_swap_test.clj+++ b/enterprise/backend/test/metabase_enterprise/replacement/source_swap_test.clj@@ -161,7 +161,7 @@query (lib/query mp (lib.metadata/table mp (mt/id :orders)))](mt/with-temp [:model/Transform {transform-id :id} {:name "test transform":source {:type "query" :query query}- :target {:database (mt/id) :table "out"}}]+ :target {:database (mt/id) :type "table" :name "out"}}](replacement.field-refs/upgrade-field-refs! [:transform transform-id])(replacement.source-swap/swap-source! [:transform transform-id][:table (mt/id :orders)]@@ -178,7 +178,7 @@(mt/with-temp [:model/Transform {transform-id :id} {:name "test transform":source {:type "query":query (lib/native-query mp "SELECT 1")}- :target {:database (mt/id) :table "out"}}]+ :target {:database (mt/id) :type "table" :name "out"}}](t2/query-one {:update :transform:set {:source "{\"type\":\"query\",\"query\":{}}"}:where [:= :id transform-id]})@@ -506,7 +506,7 @@query (lib/query mp (lib.metadata/table mp (mt/id :orders)))](mt/with-temp [:model/Transform {transform-id :id} {:name "test transform":source {:type "query" :query query}- :target {:database (mt/id) :table "out"}}]+ :target {:database (mt/id) :type "table" :name "out"}}](replacement.field-refs/upgrade-field-refs! [:transform transform-id])(mt/with-model-cleanup [:model/Dependency](events/publish-event! :event/transform-create {:object (t2/select-one :model/Transform transform-id)diff --git a/enterprise/backend/test/metabase_enterprise/sandbox/api/column_filter_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/api/column_filter_test.cljindex 752b6f6c976..8034f46d7f4 100644--- a/enterprise/backend/test/metabase_enterprise/sandbox/api/column_filter_test.clj+++ b/enterprise/backend/test/metabase_enterprise/sandbox/api/column_filter_test.clj@@ -4,6 +4,8 @@[metabase-enterprise.sandbox.api.column-filter :as col-filter][metabase-enterprise.sandbox.test-util :as mt.tu][metabase-enterprise.test :as met]+ [metabase.lib.core :as lib]+ [metabase.lib.test-metadata :as meta][metabase.test :as mt][metabase.util :as u][toucan2.core :as t2]))@@ -17,7 +19,7 @@(testing "filter-fields-by-card returns fields unchanged when card has no dataset_query"(let [fields [{:id 1 :name "A"}]](is (= fields (col-filter/filter-fields-by-card- {:dataset_query nil :result_metadata []}+ {:id 1 :dataset_query nil :result_metadata []}fields))))))(deftest empty-result-metadata-fails-closed-test@@ -33,22 +35,12 @@See the namespace docstring of metabase-enterprise.sandbox.api.column-filterfor the full rationale."- (let [mbql-card-empty-md {:dataset_query {:type :query- :query {:source-table 1}- :database 1}- :result_metadata []}- mbql-card-nil-md {:dataset_query {:type :query- :query {:source-table 1}- :database 1}- :result_metadata nil}- native-card-empty {:dataset_query {:type :native- :native {:query "SELECT 1"}- :database 1}- :result_metadata []}- native-card-nil {:dataset_query {:type :native- :native {:query "SELECT 1"}- :database 1}- :result_metadata nil}+ (let [mbql-query (lib/query meta/metadata-provider (meta/table-metadata :venues))+ native-query (lib/native-query meta/metadata-provider "SELECT 1")+ mbql-card-empty-md {:id 1 :dataset_query mbql-query :result_metadata []}+ mbql-card-nil-md {:id 2 :dataset_query mbql-query :result_metadata nil}+ native-card-empty {:id 3 :dataset_query native-query :result_metadata []}+ native-card-nil {:id 4 :dataset_query native-query :result_metadata nil}fields [{:id 1 :name "A"} {:id 2 :name "B"} {:id 3 :name "C"}]](testing "MBQL card with empty result_metadata filters everything out"(is (= [] (col-filter/filter-fields-by-card mbql-card-empty-md fields))))diff --git a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.cljindex fc9278cf964..1d34a13e21f 100644--- a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj+++ b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj@@ -63,7 +63,8 @@(qp.store/with-metadata-provider (mt/id)(sql.qp/->honeysql(or driver/*driver* :h2)- [:field {::add/source-table (mt/id table-key)+ [:field {:lib/uuid (str (random-uuid))+ ::add/source-table (mt/id table-key)::add/source-alias field-name::add/desired-alias field-name}field-id])))))@@ -137,7 +138,7 @@[:raw "{{user}}"]]:order-by [[(identifier :checkins :id) :asc]]})- :template_tags+ :template-tags{"user" {:name "user":display-name "User ID":type :number@@ -1882,7 +1883,7 @@;; To guard against a fix that simply disables the filter, the sandbox query below also joins in a column from;; another table (`orders.total`) — that one *must* still be filtered out.(let [mp (lib.tu/mock-metadata-provider- {:database {:id 1 :name "db" :dialect :h2 :engine :h2}+ {:database {:id 1 :name "db" :engine :h2}:tables [{:id 100 :name "customers" :db-id 1}{:id 200 :name "orders" :db-id 1}]:fields [{:id 1001 :name "id" :base-type :type/Integer :table-id 100diff --git a/enterprise/backend/test/metabase_enterprise/security_center/matching_test.clj b/enterprise/backend/test/metabase_enterprise/security_center/matching_test.cljindex fbf4607ab2d..29e24ba83d3 100644--- a/enterprise/backend/test/metabase_enterprise/security_center/matching_test.clj+++ b/enterprise/backend/test/metabase_enterprise/security_center/matching_test.clj@@ -104,12 +104,12 @@:where [:= :email "[email protected]"] :limit 1}}))))(testing "falls back to :default when dialect not present"(is (true? (matching/execute-matching-query!- {:some_other_db {:select [1] :from [:core_user]- :where [:= :email "[email protected]"] :limit 1}- :default {:select [1] :from [:core_user] :limit 1}}))))+ {(first (disj #{:h2 :mysql :postgres} (mdb/db-type))) {:select [1] :from [:core_user]+ :where [:= :email "[email protected]"] :limit 1}+ :default {:select [1] :from [:core_user] :limit 1}}))))(testing "no dialect match and no default → :error"(is (= :error (matching/execute-matching-query!- {:some_other_db {:select [1] :from [:core_user] :limit 1}}))))+ {(first (disj #{:h2 :mysql :postgres} (mdb/db-type))) {:select [1] :from [:core_user] :limit 1}}))))(testing "query against missing table returns :error"(is (= :error (matching/execute-matching-query!{:default {:select [1] :from [:nonexistent_table_xyz] :limit 1}}))))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.cljindex a6420418eae..22758f86a92 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj@@ -9,6 +9,7 @@[metabase-enterprise.serialization.v2.ingest :as serdes.ingest][metabase-enterprise.serialization.v2.load :as serdes.load][metabase.actions.models :as action]+ [metabase.actions.schema :as actions.schema][metabase.collections.models.collection :as collection][metabase.lib-be.core :as lib-be][metabase.lib.core :as lib]@@ -1609,12 +1610,13 @@:dataset_query {:database (:id db):type :native:native {:query "select 1"}})- _action-id (action/insert! {:entity_id eid- :name "the action"- :model_id (:id card)- :type :query- :dataset_query (mt/mbql-query users {:limit 1})- :database_id (:id db)})]+ _action-id (action/insert! (lib/normalize ::actions.schema/action.for-insert+ {:entity_id eid+ :name "the action"+ :model_id (:id card)+ :type :query+ :dataset_query (mt/mbql-query users {:limit 1})+ :database_id (:id db)}))](reset! serialized (into [] (serdes.extract/extract {:no-settings true})))(let [action-serialized (first (filter (fn [{[{:keys [model id]}] :serdes/meta}](and (= model "Action") (= id eid)))@@ -2648,7 +2650,7 @@(ts/with-db source-db(ts/create! :model/Channel :name "Test Email Channel":type :channel/email- :details {:host "smtp.example.com" :port 587}+ :details {}:description "A test email channel")(reset! serialized (into [] (serdes.extract/extract {})))(is (some (fn [{[{:keys [model id]}] :serdes/meta}]@@ -2701,7 +2703,7 @@(ts/with-db source-db(ts/create! :model/Channel :name "Minimal Channel":type :channel/email- :details {:host "smtp.example.com" :port 587}+ :details {}:description "Some description")(reset! serialized (into [] (serdes.extract/extract {}))))(let [minimal (mapv (fn [entity]diff --git a/enterprise/backend/test/metabase_enterprise/transforms_inspector/context_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_inspector/context_test.cljindex b21253dc243..1e49b9f0038 100644--- a/enterprise/backend/test/metabase_enterprise/transforms_inspector/context_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms_inspector/context_test.clj@@ -203,9 +203,10 @@(deftest build-context-mbql-sources-test(testing "build-context extracts source tables from MBQL query"- (let [transform {:source {:type :query+ (let [transform {:id 1+ :source {:type :query:query (simple-orders-query)}- :target {:schema "nonexistent" :name "nonexistent_table" :type :table}+ :target {:schema "nonexistent" :name "nonexistent_table" :type "table"}:name "test"}ctx (context/build-context transform)](is (= :mbql (:source-type ctx)))@@ -216,9 +217,10 @@(deftest build-context-not-run-test(testing "build-context returns nil target when target table doesn't exist"- (let [transform {:source {:type :query+ (let [transform {:id 1+ :source {:type :query:query (simple-orders-query)}- :target {:schema "nonexistent" :name "nonexistent_table" :type :table}+ :target {:schema "nonexistent" :name "nonexistent_table" :type "table"}:name "test"}ctx (context/build-context transform)](is (nil? (:target ctx)))@@ -227,9 +229,10 @@(deftest build-context-with-joins-test(mt/test-drivers (mt/normal-drivers-with-feature :left-join)(testing "build-context extracts join structure from MBQL query with LEFT JOIN"- (let [transform {:source {:type :query+ (let [transform {:id 1+ :source {:type :query:query (joined-orders-products-query)}- :target {:schema "nonexistent" :name "nonexistent_table" :type :table}+ :target {:schema "nonexistent" :name "nonexistent_table" :type "table"}:name "test"}ctx (context/build-context transform)](is (true? (:has-joins? ctx)))@@ -248,11 +251,12 @@(testing "build-context populates column-matches when target exists";; Use products table as a fake target to avoid running a transform(let [products-table (t2/select-one :model/Table (mt/id :products))- transform {:source {:type :query+ transform {:id 1+ :source {:type :query:query (joined-orders-products-query)}:target {:schema (:schema products-table):name (:name products-table)- :type :table}+ :type "table"}:name "test"}ctx (context/build-context transform)](is (some? (:target ctx)))diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/base_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/base_test.cljindex a8bbb5aa493..303696053b2 100644--- a/enterprise/backend/test/metabase_enterprise/transforms_python/base_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/base_test.clj@@ -69,8 +69,8 @@#'base/transfer-file-to-db (constantly nil)}(fn [](let [response (#'base/run-python-transform-impl!- {:source {:source-tables [] :body "stub"}- :target {:type "table"}+ {:source {:type :python :source-tables [] :body "stub"}+ :target {:type "table" :name "out"}:id 7}{:engine :h2 :id 1}42diff --git a/modules/drivers/athena/test/metabase/driver/athena_test.clj b/modules/drivers/athena/test/metabase/driver/athena_test.cljindex f33616232ef..8d2532030f1 100644--- a/modules/drivers/athena/test/metabase/driver/athena_test.clj+++ b/modules/drivers/athena/test/metabase/driver/athena_test.clj@@ -549,6 +549,7 @@[true? {:dbname ""}][false? {:dbname "db_name"}]]](is (schemas-supported? (driver/database-supports? :athena :schemas {:lib/type :metadata/database+ :id 1:details details})))))(deftest ^:parallel athena-describe-databasediff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.cljindex da2c4f408ed..dec2ea46fbf 100644--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj@@ -31,6 +31,7 @@[metabase.util.malli :as mu][metabase.util.performance :as perf])(:import+ (clojure.core.async.impl.channels ManyToManyChannel)(clojure.lang PersistentList)(com.google.api.gax.rpc FixedHeaderProvider)(com.google.cloud.bigquery@@ -48,6 +49,7 @@DatasetIdFieldField$Mode+ FieldListFieldValueFieldValueListJob@@ -106,7 +108,7 @@[:api-host :token-host]))(mu/defn- database-details->client- ^BigQuery [details :- :map]+ ^BigQuery [details :- :metabase.lib.schema.common/database-details](driver.u/validate-connection-hosts! :bigquery-cloud-sdk details)(let [base-creds (bigquery.common/database-details->service-account-credential details)creds (.createScoped base-creds bigquery-scopes)@@ -246,7 +248,9 @@(.getTable client dataset-id table-id empty-table-options)))(mu/defn- get-table :- (driver-api/instance-of-class Table)- (^Table [database dataset-id table-id]+ (^Table [database :- :metabase.warehouses.schema/database-or-metadata+ dataset-id :- driver-api/schema.common.non-blank-string+ table-id :- driver-api/schema.common.non-blank-string](let [details (driver.conn/effective-details database)project-id (:project-id details)](get-table (database-details->client details) project-id dataset-id table-id)))@@ -394,9 +398,11 @@[database-type (database-type->base-type database-type)]))(mu/defn- fields->metabase-field-info- ([fields]+ ([fields :- (driver-api/instance-of-class FieldList)](fields->metabase-field-info nil nil fields))- ([database-position nfc-path fields]+ ([database-position :- [:maybe :int]+ nfc-path :- [:maybe [:sequential :string]]+ fields :- (driver-api/instance-of-class FieldList)](into[](map@@ -1053,11 +1059,11 @@:ready (bigquery-execute-response result job client respond cancel-chan)))))(mu/defn- ^:dynamic *process-native*- [respond :- fn?- database :- [:map [:details :map]]- sql- parameters- cancel-chan]+ [respond :- fn?+ database :- :metabase.warehouses.schema/database-or-metadata+ sql :- :string+ parameters :- [:maybe [:sequential :metabase.lib.schema.common/field-value]]+ cancel-chan :- [:maybe (driver-api/instance-of-class ManyToManyChannel)]]{:pre [(map? database) (map? (:details database))]};; automatically retry the query if it times out or otherwise fails. This is on top of the auto-retry added by;; `execute`diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/common.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/common.cljindex 191d0f17c95..dd08da2238c 100644--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/common.clj+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/common.clj@@ -1,6 +1,7 @@(ns metabase.driver.bigquery-cloud-sdk.common"Common utility functions and utilities for the bigquery-cloud-sdk driver and related namespaces."(:require+ [malli.core :as mc][metabase.driver.bigquery-cloud-sdk.db :as bigquery.db][metabase.driver.connection :as driver.conn][metabase.util :as u]@@ -27,7 +28,7 @@(ServiceAccountCredentials/fromStream (ByteArrayInputStream. (.getBytes service-account-json))))(def ^:private RequiredDetails- [:map [:service-account-json :string]])+ [:map [:service-account-json :string] [::mc/default :metabase.lib.schema.common/database-details]])(mu/defn database-details->service-account-credential"Returns a `ServiceAccountCredentials` (not scoped) for the given `db-details`, which is based upon the value@@ -67,7 +68,7 @@Returns the calculated project-id (see [[database-details->credential-project-id]]) String from the credentials."{:added "0.42.0"}- ^String [database :- [:map [:details RequiredDetails]]]+ ^String [database :- :metabase.warehouses.schema/database-or-metadata];; :project-id-from-credentials is a database-level cache managed by this driver. We store and read it from;; `:details` regardless of connection type. This is valid so long as read and write service accounts share a;; project ID. See also: [[metabase.driver.bigquery-cloud-sdk.query-processor/project-id-for-current-query]]diff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.cljindex baf49f60549..c77b89ae0a1 100644--- a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.clj+++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.clj@@ -345,7 +345,8 @@(is (= [:=[::bigquery.qp/extract :dayofweek expected-identifier nil][:inline 1]]- (sql.qp/->honeysql :bigquery-cloud-sdk [:= {} [:field {:temporal-unit :day-of-week+ (sql.qp/->honeysql :bigquery-cloud-sdk [:= {} [:field {:lib/uuid (str (random-uuid))+ :temporal-unit :day-of-week::add/source-table "ABC"} (:id field)] 1]))))))))))(deftest reconcile-unix-timestamps-test@@ -487,7 +488,7 @@[:={}t- [:relative-datetime {} -1 :year]])]+ [:relative-datetime {:lib/uuid (str (random-uuid))} -1 :year]])](testing (format "\nclause = %s" (pr-str clause))(is (= expected-type(#'bigquery.qp/temporal-type relative-datetime)))))))))))@@ -502,7 +503,8 @@:bigquery-cloud-sdk(sql.qp/->honeysql:bigquery-cloud-sdk- [:field {:temporal-unit :week+ [:field {:lib/uuid (str (random-uuid))+ :temporal-unit :week:base-type :type/Date::add/source-table ::add/source::add/source-alias "date"@@ -549,7 +551,7 @@(is (= expected(between->sql [:between{}- [:field {::add/source-table (mt/id :checkins)} (mt/id :checkins :date)]+ [:field {:lib/uuid (str (random-uuid)), ::add/source-table (mt/id :checkins)} (mt/id :checkins :date)](t/local-date "2019-11-11")(t/local-date "2019-11-12")]))))(testing "Should be able to get temporal type from a `:field` with `:temporal-unit`"@@ -557,7 +559,8 @@(rest expected))(between->sql [:between{}- [:field {::add/source-table (mt/id :checkins)+ [:field {:lib/uuid (str (random-uuid))+ ::add/source-table (mt/id :checkins):temporal-unit :day} (mt/id :checkins :date)](t/local-date "2019-11-11")(t/local-date "2019-11-12")]))))@@ -565,7 +568,7 @@(is (= ["WHERE `date` BETWEEN ? AND ?" (t/local-date "2019-11-11") (t/local-date "2019-11-12")](between->sql [:between{}- [:field {:base-type :type/Date} "date"]+ [:field {:lib/uuid (str (random-uuid)), :base-type :type/Date} "date"](t/local-date-time "2019-11-11T12:00:00")(t/local-date-time "2019-11-12T12:00:00")]))))))))))@@ -847,9 +850,10 @@:bigquery-cloud-sdk[:={}- [:field {:temporal-unit unit+ [:field {:lib/uuid (str (random-uuid))+ :temporal-unit unit::add/source-table "ABC"} 1]- [:relative-datetime {} -1 unit]])}+ [:relative-datetime {:lib/uuid (str (random-uuid))} -1 unit]])}{:dialect ::h2x/unquoted-dialect}))))))))))(deftest filter-by-relative-date-ranges-test-2@@ -872,9 +876,10 @@:bigquery-cloud-sdk[:={}- [:field {:temporal-unit unit+ [:field {:lib/uuid (str (random-uuid))+ :temporal-unit unit::add/source-table "ABC"} 1]- [:relative-datetime {} -1 unit]])}+ [:relative-datetime {:lib/uuid (str (random-uuid))} -1 unit]])}{:dialect ::h2x/unquoted-dialect})](str/split-lines (driver/prettify-native-form :bigquery-cloud-sdk sql)))))](are [field-type unit expected-sql] (= (for [line expected-sql]diff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test/reconciliation_test_util.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test/reconciliation_test_util.cljindex d5af200f4c6..cd4405bb85b 100644--- a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test/reconciliation_test_util.clj+++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test/reconciliation_test_util.clj@@ -83,14 +83,14 @@:type :timestamp:as {:date (t/local-date "2019-12-10"):datetime (t/local-date-time "2019-12-10T14:47:00")}}- :unix-timestamp-seconds {:value [:field {::add/source-table (meta/id :checkins)} 4]+ :unix-timestamp-seconds {:value [:field {:lib/uuid (str (random-uuid)), ::add/source-table (meta/id :checkins)} 4]:type :timestamp:as (let [expected (-> [:timestamp_seconds (h2x/identifier :field "PUBLIC.CHECKINS" "unix_seconds")](h2x/with-database-type-info "timestamp"))]{:date [:date expected]:datetime [:datetime expected]:timestamp expected})}- :unix-timestamp-milliseconds {:value [:field {::add/source-table (meta/id :checkins)} 5]+ :unix-timestamp-milliseconds {:value [:field {:lib/uuid (str (random-uuid)), ::add/source-table (meta/id :checkins)} 5]:type :timestamp:as (let [expected (-> [:timestamp_millis (h2x/identifier :field "PUBLIC.CHECKINS" "unix_milliseconds")](h2x/with-database-type-info "timestamp"))]@@ -102,21 +102,24 @@{:identity identity:absolute-datetime (fn [filter-value](when (instance? java.time.temporal.Temporal filter-value)- [:absolute-datetime {} filter-value :default]))}+ [:absolute-datetime {:lib/uuid (str (random-uuid))} filter-value :default]))}:fields mock-temporal-fields:field-ref-fns{:basic (fn [field]- [:field {::add/source-table "ABC"} (:id field)])+ [:field {:lib/uuid (str (random-uuid)), ::add/source-table "ABC"} (:id field)]):default-temporal-unit (fn [field]- [:field {:temporal-unit :default+ [:field {:lib/uuid (str (random-uuid))+ :temporal-unit :default::add/source-table "ABC"} (:id field)]):base-type (fn [field]- [:field {:base-type (:base-type field)+ [:field {:lib/uuid (str (random-uuid))+ :base-type (:base-type field)::add/source-table "ABC"} (:name field)]):base-type-and-default-temporal-unit (fn [field]- [:field {:base-type (:base-type field)+ [:field {:lib/uuid (str (random-uuid))+ :base-type (:base-type field):temporal-unit :default::add/source-table "ABC"} (:name field)])}diff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/test/data/bigquery_cloud_sdk.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/test/data/bigquery_cloud_sdk.cljindex 2ec1f6be7c7..e0c8464986c 100644--- a/modules/drivers/bigquery-cloud-sdk/test/metabase/test/data/bigquery_cloud_sdk.clj+++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/test/data/bigquery_cloud_sdk.clj@@ -17,7 +17,8 @@[metabase.util.date-2 :as u.date][metabase.util.log :as log][metabase.util.malli :as mu]- [metabase.util.malli.registry :as mr])+ [metabase.util.malli.registry :as mr]+ [metabase.util.malli.schema :as ms])(:import(com.google.api.client.googleapis.jsonGoogleJsonResponseException)@@ -69,7 +70,7 @@(mu/defn test-dataset-id :- ::dataset-id"Prepend `database-name` with the hash of the db-def so we don't stomp on any other jobs running at the sametime."- [{:keys [database-name options] :as db-def}]+ [{:keys [database-name options] :as db-def} :- tx/DatabaseDefinitionSchema](cond (already-qualified? database-name) database-name(:static options) (str "sha_" (tx/hash-dataset (update db-def :optionsdissoc :static))@@ -194,7 +195,7 @@[:re #"^[A-Za-z_](\w| ){0,127}$"])(mu/defn- valid-field-name :- ValidFieldName- ^String [field-name]+ ^String [field-name :- ::lib.schema.common/non-blank-string]field-name)(defn- field-definitions->Fields [field-definitions]@@ -227,7 +228,7 @@(mu/defn- create-table![^String dataset-id :- ::lib.schema.common/non-blank-string^String table-id :- ::lib.schema.common/non-blank-string- field-definitions]+ field-definitions :- [:sequential (ms/InstanceOfClass metabase.test.data.interface.FieldDefinition)]](create-table*! dataset-id table-id field-definitions);; now verify that the Table was created(.listTables (bigquery) dataset-id (u/varargs BigQuery$TableListOption))diff --git a/modules/drivers/mongo/src/metabase/driver/mongo.clj b/modules/drivers/mongo/src/metabase/driver/mongo.cljindex c4cf34ae107..762a9516fa1 100644--- a/modules/drivers/mongo/src/metabase/driver/mongo.clj+++ b/modules/drivers/mongo/src/metabase/driver/mongo.clj@@ -19,7 +19,6 @@[metabase.driver.settings :as driver.settings][metabase.driver.util :as driver.u][metabase.lib.schema :as lib.schema]- [metabase.lib.schema.id :as lib.schema.id][metabase.util :as u][metabase.util.date-2 :as u.date][metabase.util.json :as json]@@ -487,17 +486,15 @@Each row represents leaf in sampled documents, its type and indices of keys present in the path of mongo of nestedobject."- [database :- [:map- [:id ::lib.schema.id/database]]- table :- [:map- [:name :string]]]- (let [pipeline (describe-table-pipeline {:collection-name (:name table)+ [database :- :metabase.warehouses.schema/database+ collection-name :- :string]+ (let [pipeline (describe-table-pipeline {:collection-name collection-name:sample-size (* table-rows-sample/nested-field-sample-limit 2):document-sample-depth describe-table-query-depth:leaf-limit (driver.settings/sync-leaf-fields-limit)})query {:database (:id database):type "native"- :native {:collection (:name table)+ :native {:collection collection-name:query (json/encode pipeline)}}](driver-api/process-query query fetch-dbfields-rff)))@@ -505,7 +502,7 @@[_driver database table]{:schema nil:name (:name table)- :fields (-> (fetch-dbfields database table)+ :fields (-> (fetch-dbfields database (:name table))(dbfields->ftree (driver.settings/sync-max-fields-per-table))ftree->nested-fields)})diff --git a/modules/drivers/mongo/src/metabase/driver/mongo/execute.clj b/modules/drivers/mongo/src/metabase/driver/mongo/execute.cljindex d334d097dab..526f3144df5 100644--- a/modules/drivers/mongo/src/metabase/driver/mongo/execute.clj+++ b/modules/drivers/mongo/src/metabase/driver/mongo/execute.clj@@ -46,7 +46,8 @@"Make sure there are no columns coming back from `results` that we weren't expecting. If there are, we did somethingwrong here and the query we generated is off."[columns :- ::mongo.qp/projections- first-row-col-names] ; first row col names seems to be `java.util.LinkedHashMap$LinkedKeySet` so don't try to Malli it+ ;; first row col names seems to be `java.util.LinkedHashMap$LinkedKeySet`+ first-row-col-names :- [:maybe (driver-api/instance-of-class java.util.Collection)]]{:pre [(every? string? columns) (every? string? first-row-col-names)]}(when (seq first-row-col-names)(let [expected-cols (set (for [col-name columns]@@ -84,8 +85,8 @@"Return column names we can expect in each `:row` of the results, and the `:unescaped` versions we should return inthr query result metadata."[{:keys [mbql? projections]} :- ::mongo.qp/compiled-pipeline-with-collection- query- first-row-col-names]+ query :- ::mongo.qp/pipeline+ first-row-col-names :- [:maybe (driver-api/instance-of-class java.util.Collection)]];; some of the columns may or may not come back in every row, because of course with mongo some key can be missing.;; That's ok, the logic below where we call `(mapv row columns)` will end up adding `nil` results for those columns.(if-not (and mbql? (seq projections))diff --git a/modules/drivers/mongo/src/metabase/driver/mongo/parameters.clj b/modules/drivers/mongo/src/metabase/driver/mongo/parameters.cljindex 75f8df97391..7a6c0ac872c 100644--- a/modules/drivers/mongo/src/metabase/driver/mongo/parameters.clj+++ b/modules/drivers/mongo/src/metabase/driver/mongo/parameters.clj@@ -72,7 +72,7 @@(mu/defn- substitute-one-field-filter-date-range[metadata-providerable :- ::lib.schema.metadata/metadata-providerable- {field :field, alias :alias, {value :value} :value}]+ {field :field, alias :alias, {value :value} :value} :- :metabase.lib.parameters.parse.types/field-filter](let [{:keys [start end]} (params.dates/date-string->range value {:inclusive-end? false})start-condition (when start(format "{%s: {$gte: %s}}"@@ -116,16 +116,17 @@(substitute-one-field-filter metadata-providerable field-filter)))(mu/defn- substitute-native-query-snippet- [[acc missing]+ [[acc missing] :- [:tuple [:sequential :string] [:maybe [:sequential :string]]]{:keys [content]} :- :metabase.lib.parameters.parse.types/referenced-query-snippet][(conj acc content) missing])(mu/defn- substitute-param[query :- ::lib.schema/querystage-number :- :int- param->value- [acc missing]- in-optional?+ param->value :- [:maybe [:map-of :metabase.lib.schema.common/non-blank-string+ :metabase.lib.parameters.parse.types/parsed-value]]+ [acc missing] :- [:tuple [:sequential :string] [:maybe [:sequential :string]]]+ in-optional? :- :boolean{:keys [k], :as _param} :- :metabase.lib.parameters.parse.types/param](let [v (get param->value k)](cond@@ -135,7 +136,7 @@(lib/parsed-field-filter-param? v)(let [no-value? (= (:value v) lib/parsed-param-no-value-placeholder)](cond- (params.ops/operator? (get-in v [:value :type]))+ (some-> (get-in v [:value :type]) params.ops/operator?)(let [param (:value v)field-name (if (str/blank? (:alias v))(mongo.qp/field->name query (:field v) ".")@@ -187,8 +188,9 @@(mu/defn- substitute-optional[query :- ::lib.schema/querystage-number :- :int- param->value- [acc missing]+ param->value :- [:maybe [:map-of :metabase.lib.schema.common/non-blank-string+ :metabase.lib.parameters.parse.types/parsed-value]]+ [acc missing] :- [:tuple [:sequential :string] [:maybe [:sequential :string]]]{subclauses :args} :- :metabase.lib.parameters.parse.types/optional](let [[opt-acc opt-missing] (substitute* query stage-number param->value subclauses true)](if (seq opt-missing)@@ -199,9 +201,10 @@"Returns a sequence of `[[replaced...] missing-parameters]`."[query :- ::lib.schema/querystage-number :- :int- param->value- xs- in-optional?]+ param->value :- [:maybe [:map-of :metabase.lib.schema.common/non-blank-string+ :metabase.lib.parameters.parse.types/parsed-value]]+ xs :- [:sequential :metabase.lib.parameters.parse/parsed-token]+ in-optional? :- :boolean](reduce(fn [[acc missing] x](cond@@ -223,8 +226,9 @@(mu/defn- substitute[query :- ::lib.schema/querystage-number :- :int- param->value :- [:maybe [:map-of :string :any]]- xs]+ param->value :- [:maybe [:map-of :metabase.lib.schema.common/non-blank-string+ :metabase.lib.parameters.parse.types/parsed-value]]+ xs :- [:sequential :metabase.lib.parameters.parse/parsed-token]](let [[replaced missing] (substitute* query stage-number param->value xs false)](when (seq missing)(throw (ex-info (tru "Cannot run query: missing required parameters: {0}" (set missing))@@ -235,11 +239,10 @@(mu/defn- parse-and-substitute[query :- ::lib.schema/querystage-number :- :int- param->value- x]- (if-not (string? x)- x- (substitute query stage-number param->value (lib/parse-parameters x false))))+ param->value :- [:maybe [:map-of :metabase.lib.schema.common/non-blank-string+ :metabase.lib.parameters.parse.types/parsed-value]]+ x :- :string]+ (substitute query stage-number param->value (lib/parse-parameters x false)))(mu/defn substitute-native-parameters :- ::lib.schema/query"Implementation of [[metabase.driver/substitute-native-parameters]] for MongoDB."@@ -248,6 +251,9 @@stage-number :- :int](let [param->value (params.values/stage->params-map query (lib/query-stage query stage-number))update-stage (fn [native-stage]- (perf/postwalk (partial parse-and-substitute query stage-number param->value)+ (perf/postwalk (fn [x]+ (if (string? x)+ (parse-and-substitute query stage-number param->value x)+ x))native-stage))](lib/update-query-stage query stage-number update :native update-stage)))diff --git a/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj b/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.cljindex b66feaea384..b177c74c74b 100644--- a/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj+++ b/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj@@ -56,10 +56,21 @@;; this is just a very limited schema to make sure we're generating valid queries. We should expand it more in the;; future+(mr/def ::bson-value+ "A value inside a Mongo aggregation pipeline, whose shape (BSON literals, arrays, and documents keyed by strings or keywords) the Mongo query language owns."+ [:schema {::mr/deliberately-open true+ :description "a Mongo aggregation pipeline value, shaped by the Mongo query language"}+ :any])++(mr/def ::bson-document+ "A document inside a Mongo aggregation pipeline, whose string or keyword keys the Mongo query language owns."+ [:map {:closed false, ::mr/deliberately-open true+ :description "a Mongo aggregation pipeline document, keyed by the Mongo query language"}])+(mr/def ::$project-stage[:map-of[:= "$project"]- [:map-of ::lib.schema.common/non-blank-string :any]])+ [:map-of ::lib.schema.common/non-blank-string ::bson-value]])(mr/def ::sort-spec[:map-of ::lib.schema.common/non-blank-string [:enum -1 1]])@@ -72,23 +83,21 @@(mr/def ::$match-stage[:map-of[:= "$match"]- [:map-of- [:and- [:or ::lib.schema.common/non-blank-string :keyword]- [:fn- {:error/message "not a $not condition"}- (complement #{:$not "$not"})]]- :any]])+ [:and+ ::bson-document+ [:fn+ {:error/message "not a $not condition"}+ (fn [m] (not-any? #{:$not "$not"} (keys m)))]]])(mr/def ::$group-stage[:map-of[:= "$group"]- [:map-of ::lib.schema.common/non-blank-string :any]])+ [:map-of ::lib.schema.common/non-blank-string ::bson-value]])(mr/def ::$add-fields-stage[:map-of[:= "$addFields"]- [:map-of ::lib.schema.common/non-blank-string :any]])+ [:map-of ::lib.schema.common/non-blank-string ::bson-value]])(mr/def ::$lookup-stage[:map-of@@ -96,16 +105,14 @@[:map{:closed true} ; add more stuff as needed[:from :string]- [:let [:map- [:vars {:optional true} :any]- [:in {:optional true} :any]]]+ [:let ::bson-document][:pipeline [:ref ::pipeline]][:as :string]]])(mr/def ::$unwind-stage[:map-of[:= "$unwind"]- [:map-of [:or :keyword :string] :any]])+ ::bson-document])(mr/def ::$limit-stage[:map-of@@ -125,7 +132,7 @@["sortBy" [:ref ::sort-spec]]["output" [:map-of::lib.schema.common/non-blank-string- [:map-of ::lib.schema.common/non-blank-string :any]]]+ [:map-of ::lib.schema.common/non-blank-string ::bson-value]]]["partitionBy" {:optional true} [:map-of::lib.schema.common/non-blank-string::lib.schema.common/non-blank-string]]]])@@ -157,7 +164,7 @@(lib.schema.common/instance-of-class org.bson.Document)][false[:and- :map+ ::bson-document[:fn{:error/message "map with a single key"}#(= (count %) 1)]@@ -208,6 +215,7 @@{:closed true} ; we should document anything else we add here.[:projections {:optional true} [:maybe ::projections]][:query [:ref ::pipeline]]+ [:params {:optional true} [:maybe [:sequential {:max 0} :string]]];; TODO (Cam 2026-07-17) it's not really clear if `:collection` is supposed to be in the top-level of the stage e.g.;;;; {:lib/type :mbql.stage/native, :collection "X", :native {...}}@@ -333,7 +341,8 @@(mu/defn field->name"Return a single string name for column metadata `col` For nested fields, this creates a combined qualified name."- ([metadata-providerable col]+ ([metadata-providerable :- ::lib.schema.metadata/metadata-providerable+ col :- ::lib.schema.metadata/column](field->name metadata-providerable col \.))([metadata-providerable :- ::lib.schema.metadata/metadata-providerable@@ -535,7 +544,7 @@ function(bin) {(mu/defn- days-till-start-of-first-full-week[metadata-providerable :- ::lib.schema.metadata/metadata-providerable- column]+ column :- ::bson-value](let [start-of-year (with-rvalue-temporal-bucketing metadata-providerable column :year)day-of-week-of-start-of-year (with-rvalue-temporal-bucketing metadata-providerable start-of-year :day-of-week)]{:$subtract [8 day-of-week-of-start-of-year]}))@@ -543,8 +552,8 @@ function(bin) {(mu/defn- week-of-year"Full explanation of this magic is in [[metabase.driver.sql.query-processor/week-of-year]]."[metadata-providerable :- ::lib.schema.metadata/metadata-providerable- column- mode]+ column :- ::bson-value+ mode :- [:enum :us :instance]](let [doy (with-rvalue-temporal-bucketing metadata-providerable column :day-of-year)dtsofw (binding [driver.common/*start-of-week* (case mode:us :sunday@@ -558,7 +567,7 @@ function(bin) {(mu/defn- with-rvalue-temporal-bucketing[metadata-providerable :- ::lib.schema.metadata/metadata-providerable- column+ column :- ::bson-value;; TODO (Cam 2026-07-24) apparently there's no complete schema with all the valid bucketing and truncation;; units (!) fix thisunit :- [:or@@ -693,7 +702,7 @@ function(bin) {{:$dateFromString {:dateString (str s)}})(mu/defn- absolute-datetime-or-time->rvalue- [metadata-providerable+ [metadata-providerable :- ::lib.schema.metadata/metadata-providerable[_ _opts t unit] :- [:or :mbql.clause/absolute-datetime :mbql.clause/time]](let [report-zone (t/zone-id (or (driver-api/report-timezone-id-if-supported :mongo (driver-api/database metadata-providerable))"UTC"))@@ -1341,7 +1350,9 @@ function(bin) {{$expr (->rvalue query stage-number value-clause)})(mu/defn- handle-filters :- ::compiled-pipeline- ([query stage-number pipeline-ctx]+ ([query :- ::lib.schema/query+ stage-number :- :int+ pipeline-ctx :- ::compiled-pipeline](handle-filters query stage-number pipeline-ctx (lib/filters query stage-number)))([query :- ::lib.schema/querystage-number :- :int@@ -1498,8 +1509,8 @@ function(bin) {(mu/defn- handle-join[query :- ::lib.schema/querystage-number :- :int- pipeline-ctx- {join-alias :alias, :keys [conditions stages strategy], :as join}]+ pipeline-ctx :- ::compiled-pipeline+ {join-alias :alias, :keys [conditions stages strategy], :as join} :- ::lib.schema.join/join](let [join-query (assoc query :stages stages){:keys [projections], pipeline :query, :or {projections [], pipeline []}} (mbql->native-rec join-query);; Get the mappings introduced by the source query.@@ -1619,7 +1630,8 @@ function(bin) {(apply distinct? (map first projected-fields)))]]]"Determine field projections for MBQL breakouts and aggregations. Returns a sequence of pairs like`[projected-field-name source]`."- [query stage-number]+ [query :- ::lib.schema/query+ stage-number :- :int](let [breakouts (lib/breakouts query stage-number)aggregations (lib/aggregations query stage-number)](concat@@ -1855,7 +1867,8 @@ function(bin) {(into {} (map #(get % i)) posts))))(mu/defn- order-by->$sort :- [:map-of ::lib.schema.common/non-blank-string [:enum -1 1]]- [query stage-number]+ [query :- ::lib.schema/query+ stage-number :- :int](into(ordered-map/ordered-map)(map (fn [[direction _opts field]]@@ -1872,11 +1885,11 @@ function(bin) {"window" {"documents" ["unbounded" "current"]}})(mr/def ::window-id- "TODO (Cam 2026-07-24) determine actual types of keys + values"+ "A `{breakout-name rvalue-expr}` map identifying the `_id` grouping key of a `$group` stage."[:maybe[:map-of::lib.schema.common/non-blank-string- :any]])+ ::bson-value]])(mu/defn- sort-lookup"Generates a lookup string for a particular field"@@ -1889,8 +1902,8 @@ function(bin) {(mu/defn- window-sort"Converts a `$sort` body to something that can be used in a `sortBy` clause in a`$setWindowFields` stage."- [id :- ::window-id- pairs]+ [id :- ::window-id+ pairs :- [:maybe [:sequential [:tuple ::lib.schema.common/non-blank-string [:enum -1 1]]]]](when (seq pairs)(into (ordered-map/ordered-map)(map (fn [[name dir]] [(sort-lookup id name) dir]))@@ -1922,7 +1935,7 @@ function(bin) {sort-expr (or;; if there is only one breakout, always use the user's sort order(when (= (count id) 1)- (window-sort id user-sort))+ (window-sort id (seq user-sort)));; if we don't have a temporal breakout, sort by the last breakout, but;; use the user's sort direction if specified(when-not finest-temporal-index@@ -1942,7 +1955,7 @@ function(bin) {produces a cumulative sum of those fields."[query :- ::lib.schema/querystage-number :- :int- window-vals :- [:map-of ::lib.schema.common/non-blank-string :any]+ window-vals :- [:map-of ::lib.schema.common/non-blank-string ::bson-value]id :- ::window-id];; if id is empty, we don't have any breakouts and so don't need to fiddle around with $setWindowFields(if (empty? id)@@ -2209,7 +2222,8 @@ function(bin) {(mu/defn- add-aggregation-pipeline :- ::compiled-pipeline"Generate the aggregation pipeline. Returns a sequence of maps representing each stage."- ([query stage-number]+ ([query :- ::lib.schema/query+ stage-number :- :int](add-aggregation-pipeline query stage-number {:projections [], :query []}))([query :- ::lib.schema/querystage-number :- :intdiff --git a/modules/drivers/mongo/test/metabase/driver/mongo/parameters_test.clj b/modules/drivers/mongo/test/metabase/driver/mongo/parameters_test.cljindex 8068b14a578..7ccc67af7c8 100644--- a/modules/drivers/mongo/test/metabase/driver/mongo/parameters_test.clj+++ b/modules/drivers/mongo/test/metabase/driver/mongo/parameters_test.clj@@ -15,7 +15,8 @@[metabase.query-processor.test :as qp][metabase.test :as mt][metabase.util.json :as json]- [metabase.util.malli :as mu]))+ [metabase.util.malli :as mu]+ [metabase.util.malli.schema :as ms]))(set! *warn-on-reflection* true)@@ -29,21 +30,36 @@(#'mongo.params/->utc-instant t))))))(mu/defn- optional :- :metabase.lib.parameters.parse.types/optional- [& args]+ [& args :- [:* [:or :string :metabase.lib.parameters.parse.types/param+ :metabase.lib.parameters.parse.types/optional]]](lib/parsed-optional-param args))+(def ^:private FieldFilterValue+ [:or ms/FieldValue [:sequential ms/FieldValue]])++(def ^:private FieldFilterOptions+ [:maybe [:map {:closed true} [:case-sensitive {:optional true} :boolean]]])+(mu/defn- field-filter :- :metabase.lib.parameters.parse.types/field-filter- ([field-name value-type value]+ ([field-name :- :string+ value-type :- :keyword+ value :- FieldFilterValue](field-filter field-name nil value-type value))- ([field-name base-type value-type value]+ ([field-name :- :string+ base-type :- [:maybe :keyword]+ value-type :- :keyword+ value :- FieldFilterValue](field-filter field-name base-type value-type value nil))- ([field-name base-type value-type value options]- (merge (lib/parsed-field-filter-param {:lib/type :metadata/column- :name (name field-name)- :base-type (or base-type :type/*)}- (cond-> {:type value-type, :value value}- (map? options) (assoc :options options)))- options)))+ ([field-name :- :string+ base-type :- [:maybe :keyword]+ value-type :- :keyword+ value :- FieldFilterValue+ options :- FieldFilterOptions]+ (lib/parsed-field-filter-param {:lib/type :metadata/column+ :name (name field-name)+ :base-type (or base-type :type/*)}+ (cond-> {:type value-type, :value value}+ (map? options) (assoc :options options)))))(defn- substitute [param->value x](#'mongo.params/substitutediff --git a/modules/drivers/mongo/test/metabase/driver/mongo/query_processor_test.clj b/modules/drivers/mongo/test/metabase/driver/mongo/query_processor_test.cljindex 3c15fa8f44f..174aa0437f5 100644--- a/modules/drivers/mongo/test/metabase/driver/mongo/query_processor_test.clj+++ b/modules/drivers/mongo/test/metabase/driver/mongo/query_processor_test.clj@@ -47,7 +47,7 @@(is (= nil(query->collection-name {:query {:source-query{:native []}- :joins [{:source-query {:source-table 1, :collection "wow"}+ :joins [{:source-query {:native [], :collection "wow"}:condition [:= 1 2]}]}}))))))(deftest ^:parallel order-postprocessing-testdiff --git a/modules/drivers/mongo/test/metabase/driver/mongo_test.clj b/modules/drivers/mongo/test/metabase/driver/mongo_test.cljindex 868d5d8f61d..58dabd903b3 100644--- a/modules/drivers/mongo/test/metabase/driver/mongo_test.clj+++ b/modules/drivers/mongo/test/metabase/driver/mongo_test.clj@@ -1050,7 +1050,7 @@(binding [mongo.execute/*aggregate* (fn [db _coll session stages timeout-ms](mongo.execute/aggregate-database db session stages timeout-ms))mongo/*sample-stages* (fn [& _#] [{"$documents" documents}])]- (let [dbfields (delay (@#'mongo/fetch-dbfields (mt/db) (t2/select-one :model/Table :id (mt/id :venues))))+ (let [dbfields (delay (@#'mongo/fetch-dbfields (mt/db) (t2/select-one-fn :name :model/Table :id (mt/id :venues))))ftree (delay (@#'mongo/dbfields->ftree @dbfields))nested-fields (delay (@#'mongo/ftree->nested-fields @ftree))](thunk dbfields ftree nested-fields)))))diff --git a/modules/drivers/oracle/src/metabase/driver/oracle.clj b/modules/drivers/oracle/src/metabase/driver/oracle.cljindex 3b601afdc70..6dbf4cffbbe 100644--- a/modules/drivers/oracle/src/metabase/driver/oracle.clj+++ b/modules/drivers/oracle/src/metabase/driver/oracle.clj@@ -5,6 +5,7 @@[clojure.string :as str][honey.sql :as sql][java-time.api :as t]+ [malli.core :as mc][metabase.driver :as driver][metabase.driver-api.core :as driver-api][metabase.driver.common :as driver.common]@@ -112,7 +113,8 @@[:ssl-truststore-value {:optional true} [:maybe string?]][:ssl-truststore-path {:optional true} [:maybe string?]][:ssl-truststore-options {:optional true} [:maybe string?]]- [:ssl-truststore-password-value {:optional true} [:maybe string?]]])+ [:ssl-truststore-password-value {:optional true} [:maybe string?]]+ [::mc/default :metabase.lib.schema.common/database-details]]);;; Everything after the `@` in `jdbc:oracle:thin:@...` is a *connect descriptor*, which the driver reads as;;; instructions and not merely as an address: a scheme there sends it to fetch a URL, read a file, or ask an LDAP@@ -236,9 +238,24 @@[_ column-type](database-type->base-type column-type))+(mr/def ::spec+ [:map {:closed true}+ [:classname :string]+ [:subprotocol :string]+ ["v$session.program" :string]+ [:user {:optional true} [:maybe :string]]+ [:password {:optional true} [:maybe :string]]+ [:javax.net.ssl.keyStore {:optional true} [:maybe (driver-api/instance-of-class java.io.File)]]+ [:javax.net.ssl.keyStorePassword {:optional true} [:maybe :string]]+ [:javax.net.ssl.keyStoreType {:optional true} [:maybe :string]]+ [:javax.net.ssl.trustStore {:optional true} [:maybe (driver-api/instance-of-class java.io.File)]]+ [:javax.net.ssl.trustStorePassword {:optional true} [:maybe :string]]+ [:javax.net.ssl.trustStoreType {:optional true} [:maybe :string]]+ [:oracle.net.authentication_services {:optional true} :string]])+;;; both take the values [[validated-details]] returned, which is what makes it safe to concatenate them(mu/defn- non-ssl-spec :- :map- [spec :- :map+ [spec :- ::spechost :- :stringport :- :intsid :- [:maybe :string]@@ -251,7 +268,7 @@(str "/" service-name)))))(mu/defn- ssl-spec :- :map- [spec :- :map+ [spec :- ::spechost :- :stringport :- :intsid :- [:maybe :string]diff --git a/modules/drivers/oracle/test/metabase/driver/oracle_test.clj b/modules/drivers/oracle/test/metabase/driver/oracle_test.cljindex dd7efb2d6b1..2cfed3c0c4a 100644--- a/modules/drivers/oracle/test/metabase/driver/oracle_test.clj+++ b/modules/drivers/oracle/test/metabase/driver/oracle_test.clj@@ -287,13 +287,13 @@:value "local"}{:name "Uploaded file path":value "uploaded"}]- :visible-if {:ssl-use-keystore true}}+ :visible-if {"ssl-use-keystore" true}}{:name "ssl-keystore-value":type "textFile"- :visible-if {:ssl-keystore-options "uploaded"}}+ :visible-if {"ssl-keystore-options" "uploaded"}}{:name "ssl-keystore-path":type "string"- :visible-if {:ssl-keystore-options "local"}}+ :visible-if {"ssl-keystore-options" "local"}}{:name "ssl-keystore-password-value":type "password"}{:name "ssl-use-truststore"}@@ -303,13 +303,13 @@:value "local"}{:name "Uploaded file path":value "uploaded"}]- :visible-if {:ssl-use-truststore true}}+ :visible-if {"ssl-use-truststore" true}}{:name "ssl-truststore-value":type "textFile"- :visible-if {:ssl-truststore-options "uploaded"}}+ :visible-if {"ssl-truststore-options" "uploaded"}}{:name "ssl-truststore-path":type "string"- :visible-if {:ssl-truststore-options "local"}}+ :visible-if {"ssl-truststore-options" "local"}}{:name "ssl-truststore-password-value":type "password"}{:name "tunnel-enabled"}@@ -329,11 +329,11 @@:visible-if {"tunnel-enabled" true}}{:name "tunnel-known-hosts-value":type "textFile"- :visible-if {:tunnel-known-hosts-options "uploaded"+ :visible-if {"tunnel-known-hosts-options" "uploaded""tunnel-enabled" true}}{:name "tunnel-known-hosts-path":type "string"- :visible-if {:tunnel-known-hosts-options "local"+ :visible-if {"tunnel-known-hosts-options" "local""tunnel-enabled" true}}{:name "advanced-options"}{:name "destination-database"}@@ -432,7 +432,9 @@:database-type "char":base-type :type/Text}]})(let [hsql (sql.qp/mbql->honeysql :oracle- {:query {:source-query {:source-table 1+ {:database 1+ :type :query+ :query {:source-query {:source-table 1:expressions {"s" [:substring [:field 1 nil] 2]}:fields [[:field 1 nil][:expression "s"]]}diff --git a/modules/drivers/redshift/test/metabase/driver/redshift_test.clj b/modules/drivers/redshift/test/metabase/driver/redshift_test.cljindex ee0c04e193e..05f44f2da66 100644--- a/modules/drivers/redshift/test/metabase/driver/redshift_test.clj+++ b/modules/drivers/redshift/test/metabase/driver/redshift_test.clj@@ -699,7 +699,7 @@:base-type {:native "VARBYTE"}:effective-type :type/Text}{:field-name "multi_driver"- :base-type {:natives {:redshift "SUPER" :postgres "JSONB"}}+ :base-type {:natives {"redshift" "SUPER" "postgres" "JSONB"}}:effective-type :type/JSON}]}]}rows (@#'test.get-or-create/dbdef->fake-sync-rows :redshift 456 dbdef)fields (:field-rows (first rows))]diff --git a/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj b/modules/drivers/snowflake/test/metabase/driver/snowflake_test.cljindex 67d35460bd2..1850c48beb6 100644--- a/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj+++ b/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj@@ -301,7 +301,7 @@(testing "the simple-select-probe-query used by have-select-privilege? should be qualified with the Database name. Ignore blank keys."(mt/test-driver :snowflake(qp.store/with-metadata-provider (lib.tu/mock-metadata-provider- {:database (assoc (mt/db)+ {:database (assoc (lib.metadata/database (mt/metadata-provider)):details {:db " ":dbname "dbname"})})(is (= ["SELECT TRUE AS \"_\" FROM \"PUBLIC\".\"table\" WHERE 1 <> 1 LIMIT 0"]diff --git a/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj b/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.cljindex 7d52a6e98f4..89d90e69ac8 100644--- a/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj+++ b/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj@@ -946,7 +946,7 @@:where (sql.qp/->honeysql:sqlserver[:= {}- [:field {} (mt/id :attempts :datetime)]+ [:field {:lib/uuid (str (random-uuid))} (mt/id :attempts :datetime)](sql.qp/compiled [:raw "?"])])})))](doseq [param [datetime-string datetime-localdatetime]:let [query [base-query param]]]@@ -967,18 +967,22 @@(testing "SQL Server default database role handling"(testing "returns role when explicitly configured"(let [database {:lib/type :metadata/database+ :id 1:details {:user "login_user" :role "db_user"}}](is (= "db_user" (driver.sql/default-database-role :sqlserver database)))))(testing "returns nil when no role is configured"(let [database {:lib/type :metadata/database+ :id 1:details {:user "login_user"}}](is (nil? (driver.sql/default-database-role :sqlserver database)))))(testing "returns nil even when user is 'sa'"(let [database {:lib/type :metadata/database+ :id 1:details {:user "sa"}}](is (nil? (driver.sql/default-database-role :sqlserver database)))))(testing "ignores user field and only uses role field"(let [database {:lib/type :metadata/database+ :id 1:details {:user "login_user" :role "impersonation_user"}}](is (= "impersonation_user" (driver.sql/default-database-role :sqlserver database)))))))diff --git a/modules/drivers/vertica/test/metabase/test/data/vertica.clj b/modules/drivers/vertica/test/metabase/test/data/vertica.cljindex 8861f98adfc..f6d33f3563d 100644--- a/modules/drivers/vertica/test/metabase/test/data/vertica.clj+++ b/modules/drivers/vertica/test/metabase/test/data/vertica.clj@@ -155,8 +155,8 @@"Load rows from a CSV file into a Table."[driver :- :keywordconn :- (lib.schema.common/instance-of-class java.sql.Connection)- {:keys [database-name], :as _dbdef} :- [:map [:database-name :string]]- {:keys [table-name rows], :as _tabledef} :- [:map [:table-name :string]]+ {:keys [database-name], :as _dbdef} :- tx/DatabaseDefinitionSchema+ {:keys [table-name rows], :as _tabledef} :- tx/TableDefinitionSchemafilename :- :string](let [table-identifier (sql.tx/qualify-and-quote driver database-name table-name)](letfn [(execute! [sql]diff --git a/resources/user_key_value_types/data_studio.edn b/resources/user_key_value_types/data_studio.ednindex 349f7ef36a3..3fcb3772c12 100644--- a/resources/user_key_value_types/data_studio.edn+++ b/resources/user_key_value_types/data_studio.edn@@ -1,2 +1,2 @@-[:map+[:map {:closed true}[:value :boolean]]diff --git a/resources/user_key_value_types/dependency_diagnostics.edn b/resources/user_key_value_types/dependency_diagnostics.ednindex 469a2f0c9e0..78fdb007a2c 100644--- a/resources/user_key_value_types/dependency_diagnostics.edn+++ b/resources/user_key_value_types/dependency_diagnostics.edn@@ -1,5 +1,5 @@-[:map- [:value [:map+[:map {:closed true}+ [:value [:map {:closed true}[:group_types {:optional true} [:sequential [:enum "question" "model" "metric" "table" "transform" "snippet" "dashboard" "document" "sandbox" "segment" "measure"]]][:include_personal_collections {:optional true} :boolean][:sort_column {:optional true} [:enum "name" "location" "dependents-errors" "dependents-with-errors"]]diff --git a/resources/user_key_value_types/embedding_hub.edn b/resources/user_key_value_types/embedding_hub.ednindex 349f7ef36a3..3fcb3772c12 100644--- a/resources/user_key_value_types/embedding_hub.edn+++ b/resources/user_key_value_types/embedding_hub.edn@@ -1,2 +1,2 @@-[:map+[:map {:closed true}[:value :boolean]]diff --git a/resources/user_key_value_types/indicator-menu.edn b/resources/user_key_value_types/indicator-menu.ednindex a5181941fba..ac107c5569f 100644--- a/resources/user_key_value_types/indicator-menu.edn+++ b/resources/user_key_value_types/indicator-menu.edn@@ -1,2 +1,2 @@-[:map+[:map {:closed true}[:value [:vector :string]]]diff --git a/resources/user_key_value_types/last_download_format.edn b/resources/user_key_value_types/last_download_format.ednindex 0f7db3790d6..67e133947e9 100644--- a/resources/user_key_value_types/last_download_format.edn+++ b/resources/user_key_value_types/last_download_format.edn@@ -1,4 +1,4 @@-[:map- [:value [:map+[:map {:closed true}+ [:value [:map {:closed true}[:last_download_format [:enum "csv" "xlsx" "json" "png"]][:last_table_download_format [:enum "csv" "xlsx" "json"]]]]]diff --git a/resources/user_key_value_types/monitor.edn b/resources/user_key_value_types/monitor.ednindex 349f7ef36a3..3fcb3772c12 100644--- a/resources/user_key_value_types/monitor.edn+++ b/resources/user_key_value_types/monitor.edn@@ -1,2 +1,2 @@-[:map+[:map {:closed true}[:value :boolean]]diff --git a/resources/user_key_value_types/schema_viewer.edn b/resources/user_key_value_types/schema_viewer.ednindex d03543f7358..689721ecd2b 100644--- a/resources/user_key_value_types/schema_viewer.edn+++ b/resources/user_key_value_types/schema_viewer.edn@@ -1,9 +1,9 @@-[:map+[:map {:closed true}[:value [:or;; Per-database/schema preferences (key = "databaseId:schema")- [:map+ [:map {:closed true}[:table_ids {:optional true} [:sequential :int]]];; Last opened database (key = "last_database")- [:map+ [:map {:closed true}[:databaseId :int][:schema {:optional true} :string]]]]]diff --git a/resources/user_key_value_types/transforms.edn b/resources/user_key_value_types/transforms.ednindex ad34f4ecd79..efb5289f91f 100644--- a/resources/user_key_value_types/transforms.edn+++ b/resources/user_key_value_types/transforms.edn@@ -1,2 +1,2 @@-[:map+[:map {:closed true}[:value [:enum "tree" "alphabetical" "last-modified"]]]diff --git a/resources/user_key_value_types/user_acknowledgement.edn b/resources/user_key_value_types/user_acknowledgement.ednindex 349f7ef36a3..3fcb3772c12 100644--- a/resources/user_key_value_types/user_acknowledgement.edn+++ b/resources/user_key_value_types/user_acknowledgement.edn@@ -1,2 +1,2 @@-[:map+[:map {:closed true}[:value :boolean]]diff --git a/src/metabase/actions/actions.clj b/src/metabase/actions/actions.cljindex 49910307588..89b493c2570 100644--- a/src/metabase/actions/actions.clj+++ b/src/metabase/actions/actions.clj@@ -9,6 +9,7 @@[metabase.actions.hierarchy :as actions.hierarchy][metabase.actions.scope :as actions.scope][metabase.actions.settings :as actions.settings]+ [metabase.actions.types :as actions.types][metabase.api.common :as api][metabase.driver :as driver][metabase.driver.connection :as driver.conn]@@ -16,6 +17,8 @@[metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]+ [metabase.lib.schema.actions :as lib.schema.actions]+ [metabase.lib.schema.id :as lib.schema.id][metabase.query-processor.middleware.permissions :as qp.perms];; legacy usage -- don't do things like this going forward^{:clj-kondo/ignore [:deprecated-namespace :discouraged-namespace]} [metabase.query-processor.store :as qp.store]@@ -25,6 +28,7 @@[metabase.util.log :as log][metabase.util.malli :as mu][metabase.util.malli.registry :as mr]+ [metabase.util.malli.schema :as ms][methodical.core :as methodical])(:import(clojure.lang ExceptionInfo)))@@ -172,12 +176,27 @@(doseq [[event-type payloads] (u/group-by first second effects)](handle-effects!* event-type sans-effects payloads))))+(def ^:private RowDiff+ [:map {:closed true}+ [:table-id ::lib.schema.id/table]+ [:db-id ::lib.schema.id/database]+ [:before [:maybe ::lib.schema.actions/row]]+ [:after [:maybe ::lib.schema.actions/row]]])++(def ^:private ActionContext+ [:map {:closed true}+ [:user-id {:optional true} [:maybe ms/PositiveInt]]+ [:scope {:optional true} [:maybe ::actions.types/scope.hydrated]]+ [:driver {:optional true} [:maybe :keyword]]+ [:invocation-id {:optional true} [:maybe :string]]+ [:invocation-stack {:optional true} [:maybe [:sequential [:tuple qualified-keyword? :string]]]]+ [:effects {:optional true} [:maybe [:sequential [:tuple qualified-keyword? RowDiff]]]]])+(mu/defn- perform-action-internal![action-kw :- qualified-keyword?- ctx :- :map+ ctx :- ActionContext;; Since the inner map shape will depend on action-kw, we will need to dynamically validate it.- inputs :- [:sequential :map]- & {:as _opts}]+ inputs :- [:sequential ::actions.args/any-arg-map]](driver.conn/with-write-connection(lib-be/with-metadata-provider-cache(let [invocation-id (u/generate-nano-id)@@ -241,7 +260,7 @@(mu/defn- check-permissions[policy :- :keywordarg-maps :- [:sequential [:or- ::actions.args/common+ ::actions.args/any-arg-map[:= {:description "empty map"} {}]]]](when (#{:model-action :ad-hoc-invocation} policy)(doseq [arg-map arg-maps@@ -252,18 +271,28 @@(lib-be/normalize-query arg-map))]](qp.perms/check-query-action-permissions* query))))+(def ^:private PerformActionOpts+ [:map {:closed true}+ [:policy {:optional true} [:maybe [:enum :model-action :ad-hoc-invocation :data-editing]]]+ [:existing-context {:optional true} [:maybe ActionContext]]+ [:user-id {:optional true} [:maybe ms/PositiveInt]]+ [:action-id {:optional true} [:maybe ms/PositiveInt]]+ [:dashboard-id {:optional true} [:maybe ms/PositiveInt]]+ [:context {:optional true} [:maybe [:enum :action-execute :public-action-execute]]]])+;; TODO rename this to just perform-action! and rename the legacy entry point to clearly deprecate it.(mu/defn perform-action-v2!"Perform an *implicit* `action`. This is the main entry point that handles validation, permissions, and more.Implement [[perform-action!*]] to add support for a new driver/action combo.The shape of `arg-map` depends on the `action` being performed. "- [action- scope- arg-map-or-maps+ [action :- [:or :keyword ms/NonBlankString]+ scope :- ::actions.types/scope.raw+ arg-map-or-maps :- [:or ::actions.args/any-arg-map [:sequential ::actions.args/any-arg-map]];; `action-id`, `dashboard-id` and `audit-context` are attribution for the audit row; the scope maps are closed;; schemas, so they ride along as kwargs instead.& {:keys [policy existing-context user-id action-id dashboard-id]- audit-context :context}]+ audit-context :context}+ :- [:maybe PerformActionOpts]](when (and existing-context user-id)(assert (= user-id (:user-id existing-context)) "Existing context has a consistent user-id"))(log/with-context {:action action}@@ -348,7 +377,10 @@(mu/defn perform-action!"This is the Old School version of [[perform-action!], before we returned effects and added generic bulk application."- [action arg-map & {:keys [scope] :as opts}]+ [action :- [:or :keyword ms/NonBlankString]+ arg-map :- ::actions.args/any-arg-map+ & {:keys [scope] :as opts}+ :- [:maybe [:merge PerformActionOpts [:map [:scope {:optional true} [:maybe ::actions.types/scope.raw]]]]]](try (let [scope (or scope {:unknown :model-action}){:keys [outputs]} (perform-action-v2! action scope [arg-map] (dissoc opts :scope))](assert (= 1 (count outputs)) "The legacy action APIs do not support actions with multiple outputs")diff --git a/src/metabase/actions/args.clj b/src/metabase/actions/args.cljindex 48b8db353fb..65832617001 100644--- a/src/metabase/actions/args.clj+++ b/src/metabase/actions/args.clj@@ -52,7 +52,7 @@;;; Anything else required depends on the action type.(mr/def ::common- [:map [:database ::lib.schema.id/database]])+ [:map {:closed true} [:database ::lib.schema.id/database]])(mr/def ::row[:map-of :string [:ref ::lib.schema.parameter/parameter.value]])@@ -62,12 +62,19 @@;;; {:database <id>, :query {:source-table <id>}}(mr/def ::query- [:map [:source-table ::lib.schema.id/table]])+ [:map {:closed true} [:source-table ::lib.schema.id/table]])++(mr/def ::query.filtered+ [:map {:closed true}+ [:source-table ::lib.schema.id/table]+ [:filter :metabase.legacy-mbql.schema/Filter]])(mr/def ::crud.row.common[:merge::common- [:map [:query ::query]]])+ [:map+ [:type {:optional true} [:= :query]]+ [:query ::query]]]);;;; `:model.row/create`@@ -102,9 +109,7 @@[:merge::crud.row.common[:map [:update-row ::row]- [:query [:merge- ::query- [:map [:filter [:sequential :any]]]]]]])+ [:query ::query.filtered]]])(defmethod action-arg-map-schema :model.row/update[_action]@@ -124,9 +129,7 @@(mr/def ::model.row.delete[:merge::crud.row.common- [:map [:query [:merge- ::query- [:map [:filter [:sequential :any]]]]]]])+ [:map [:query ::query.filtered]]])(defmethod action-arg-map-schema :model.row/delete[_action]@@ -146,7 +149,8 @@[:merge::common[:map [:table-id pos-int?]- [:row ::row]]])+ [:row {:optional true} ::row]+ [:arg {:optional true} ::row]]]);;; The request bodies for the table CRUD actions are all the same. The body of a request to `POST;;; /api/action/:action-namespace/:action-name/:table-id` is just a vector of rows but the API endpoint itself calls@@ -176,3 +180,35 @@{:database (or database (when table-id (actions.db/table-database-id table-id))):table-id table-id:row (update-keys (or row row-arg) u/qualified-name)})++(mr/def ::implicit+ [:map {:closed true}+ [:database ::lib.schema.id/database]+ [:type [:= :query]]+ [:query [:or ::query ::query.filtered]]+ [:create-row {:optional true} ::row]+ [:update-row {:optional true} ::row]])++(mr/def ::table.insert+ [:map {:closed true}+ [:database ::lib.schema.id/database]+ [:table-id ::lib.schema.id/table]+ [:values ::row]])++(mr/def ::data-grid.row.input+ "A data-grid row action input before its normalization resolves the database."+ [:map {:closed true}+ [:table-id ::lib.schema.id/table]+ [:row ::row]])++(mr/def ::any-arg-map+ "One arg map an action can be invoked with, in any of the shapes [[action-arg-map-schema]] recognizes."+ [:or+ ::model.row.create+ ::model.row.update+ ::model.row.delete+ ::table.common+ ::data-grid.row.input+ ::implicit+ ::table.insert+ [:= {} {}]])diff --git a/src/metabase/actions/audit.clj b/src/metabase/actions/audit.cljindex da0eed0b604..490c2917613 100644--- a/src/metabase/actions/audit.clj+++ b/src/metabase/actions/audit.clj@@ -8,10 +8,15 @@invocation is one row."(:require[java-time.api :as t]+ [metabase.actions.args :as actions.args][metabase.actions.db :as actions.db]+ [metabase.actions.types :as actions.types][metabase.analytics.settings :as analytics.settings][metabase.api.common :as api][metabase.lib-be.core :as lib-be]+ [metabase.lib-be.schema :as lib-be.schema]+ [metabase.lib.schema :as lib.schema]+ [metabase.lib.schema.parameter :as lib.schema.parameter][metabase.permissions.core :as perms][metabase.queries.models.query :as query][metabase.util :as u]@@ -21,6 +26,27 @@(set! *warn-on-reflection* true)+(def ^:private InternalTemplate+ "An action descriptor `:template` -- identifies the action, never its content."+ [:map {:closed true}+ [:type [:= :internal]]+ [:action [:or :string :keyword]]+ [:database {:optional true} [:maybe pos-int?]]+ [:scope {:optional true} ::actions.types/scope.normalized]+ [:action-id {:optional true} [:maybe pos-int?]]])++(def ^:private Inputs+ "The input values that were actually supplied to an action invocation, across the various shapes an action's+ arguments can take."+ [:sequential [:or+ [:= {} {}]+ ::lib.schema.parameter/parameter+ [:map-of ::lib.schema.parameter/id ::lib.schema.parameter/parameter.value]+ ::actions.args/model.row.create+ ::actions.args/model.row.update+ ::actions.args/model.row.delete+ ::actions.args/table.common]])+(def ^:private Base"What a recording site knows about an invocation before it runs."[:map {:closed true}@@ -33,9 +59,9 @@[:context [:enum :action-execute :public-action-execute]][:native? :boolean];; hashed, and stored in `query` -- the SQL template or an action descriptor, never the input values- [:template :map]+ [:template [:or ::lib.schema/query ::lib-be.schema/maybe-legacy-or-empty-query InternalTemplate]];; the input values that were actually supplied: PII-gated into `parameters`- [:inputs [:sequential :any]]])+ [:inputs Inputs]])(defn- impersonated?"Whether an enforced connection-impersonation policy applies to the current user on `database-id`: the question thediff --git a/src/metabase/actions/db.clj b/src/metabase/actions/db.cljindex c56501971a9..168a82e42f8 100644--- a/src/metabase/actions/db.clj+++ b/src/metabase/actions/db.clj@@ -51,7 +51,7 @@(mu/defn table"The Table with `table-id`, or nil."- [table-id :- ::lib.schema.id/table]+ [table-id :- [:maybe ::lib.schema.id/table]](t2/select-one :model/Table :id table-id {:from [(warehouse-schema-overlay/table-query)]}))(mu/defn tablesdiff --git a/src/metabase/actions/execution.clj b/src/metabase/actions/execution.cljindex b27fc29f63e..f6ebd75b0c8 100644--- a/src/metabase/actions/execution.clj+++ b/src/metabase/actions/execution.clj@@ -8,13 +8,14 @@[metabase.actions.db :as actions.db][metabase.actions.http-action :as http-action][metabase.actions.models :as action]+ [metabase.actions.schema :as actions.schema][metabase.analytics.core :as analytics][metabase.api.common :as api][metabase.driver.connection :as driver.conn];; legacy usage, do not use this in new code^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.schema :as mbql.s]- [metabase.lib.schema :as lib.schema][metabase.lib.schema.id :as lib.schema.id]+ [metabase.lib.schema.parameter :as lib.schema.parameter][metabase.model-persistence.core :as model-persistence][metabase.parameters.schema :as parameters.schema][metabase.queries.models.query :as query]@@ -26,18 +27,27 @@[metabase.util.i18n :refer [tru]][metabase.util.log :as log][metabase.util.malli :as mu]+ [metabase.util.malli.schema :as ms][toucan2.core :as t2]))+(def ^:private RequestParameters+ "Parameters as passed in from an endpoint, of shape `{<parameter-id> <value>}`."+ [:maybe [:map-of ::lib.schema.parameter/id ::lib.schema.parameter/parameter.value]])++(def ^:private ExecuteActionOpts+ [:map {:closed true}+ [:allow-http-actions? {:optional true} [:maybe :boolean]]+ [:context {:optional true} [:maybe :keyword]]+ [:dashboard-id {:optional true} [:maybe ms/PositiveInt]]])+(mu/defn- execute-query-action!"Execute a `QueryAction` with parameters as passed in from anendpoint of shape `{<parameter-id> <value>}`.`action` should already be hydrated with its `:card`. `opts` carries the audit attribution from the endpoint."- [{query :dataset_query, model-id :model_id, :as action} :- [:map- [:model_id ::lib.schema.id/card]- [:dataset_query ::lib.schema/native-only-query]]- request-parameters- opts]+ [{query :dataset_query, model-id :model_id, :as action} :- ::actions.schema/action+ request-parameters :- RequestParameters+ opts :- [:maybe ExecuteActionOpts]](log/tracef "Executing action for model %d" model-id)(driver.conn/with-write-connection(try@@ -147,11 +157,16 @@:row/update :model.row/update:row/delete :model.row/delete})+(def ^:private ImplicitActionKind+ [:enum :model.row/create :model.row/update :model.row/delete :bulk/create :bulk/update :bulk/delete])+(mu/defn- build-implicit-query :- [:map[:query ::mbql.s/Query][:row-parameters ::actions.args/row][:prefetch-parameters {:optional true} [:maybe ::parameters.schema/parameters]]]- [{:keys [model_id parameters] :as _action} implicit-action request-parameters]+ [{:keys [model_id parameters] :as _action} :- ::actions.schema/action+ implicit-action :- ImplicitActionKind+ request-parameters :- RequestParameters](let [{database-id :db_idtable-id :id :as table} (implicit-action-table model_id)table-fields (:fields table)@@ -229,9 +244,12 @@(mu/defn execute-action!"Execute the given action with the given parameters of shape `{<parameter-id> <value>}."- ([action request-parameters]+ ([action :- ::actions.schema/action+ request-parameters :- RequestParameters](execute-action! action request-parameters nil))- ([action request-parameters {:keys [allow-http-actions?] :or {allow-http-actions? true} :as opts}]+ ([action :- ::actions.schema/action+ request-parameters :- RequestParameters+ {:keys [allow-http-actions?] :or {allow-http-actions? true} :as opts} :- [:maybe ExecuteActionOpts]](when (and (= (:type action) :http) (not allow-http-actions?))(throw (ex-info (tru "HTTP actions cannot be executed from public endpoints."){:status-code 403})))@@ -262,12 +280,14 @@(mu/defn execute-dashcard!"Execute the given action in the dashboard/dashcard context with the given parametersof shape `{<parameter-id> <value>}."- ([dashboard-id dashcard-id request-parameters]+ ([dashboard-id :- ::lib.schema.id/dashboard+ dashcard-id :- ::lib.schema.id/dashcard+ request-parameters :- RequestParameters](execute-dashcard! dashboard-id dashcard-id request-parameters nil))([dashboard-id :- ::lib.schema.id/dashboarddashcard-id :- ::lib.schema.id/dashcard- request-parameters :- [:maybe [:map-of :string :any]]- opts]+ request-parameters :- RequestParameters+ opts :- [:maybe ExecuteActionOpts]](let [dashcard (api/check-404 (actions.db/dashcard-in-dashboard dashcard-id dashboard-id))action (api/check-404 (action/select-action :id (:action_id dashcard)))](analytics/track-event! :snowplow/actiondiff --git a/src/metabase/actions/http_action.clj b/src/metabase/actions/http_action.cljindex b4c284bc61d..941cfc9fdc9 100644--- a/src/metabase/actions/http_action.clj+++ b/src/metabase/actions/http_action.clj@@ -7,7 +7,8 @@[metabase.util.i18n :refer [tru]][metabase.util.json :as json][metabase.util.log :as log]- [metabase.util.malli :as mu])+ [metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr])(:import(com.fasterxml.jackson.databind ObjectMapper)(net.thisptr.jackson.jq BuiltinFunctionLoader JsonQuery Output Scope Versions)))@@ -26,11 +27,14 @@;; May go away if parameters substitution is taken out of query-processing/db dependency(declare substitute*)+(mr/def ::acc+ [:tuple [:maybe :string] [:maybe [:sequential :string]]])+(mu/defn- substitute-param- [param->value- [sql missing]- _in-optional?- {:keys [k]} :- :metabase.lib.parameters.parse.types/param]+ [param->value :- [:maybe [:map-of :string :metabase.lib.parameters.parse.types/parsed-value]]+ [sql missing] :- ::acc+ _in-optional? :- :boolean+ {:keys [k]} :- :metabase.lib.parameters.parse.types/param](if-not (contains? param->value k)[sql (conj missing k)](let [v (get param->value k)]@@ -42,8 +46,8 @@[(str sql v) missing]))))(mu/defn- substitute-optional- [param->value- [sql missing]+ [param->value :- [:maybe [:map-of :string :metabase.lib.parameters.parse.types/parsed-value]]+ [sql missing] :- ::acc{subclauses :args} :- :metabase.lib.parameters.parse.types/optional](let [[opt-sql opt-missing] (substitute* param->value subclauses true)](if (seq opt-missing)diff --git a/src/metabase/actions/models.clj b/src/metabase/actions/models.cljindex 91665ee7fd3..1db38af9933 100644--- a/src/metabase/actions/models.clj+++ b/src/metabase/actions/models.clj@@ -5,15 +5,19 @@[metabase.actions.schema :as actions.schema][metabase.lib-be.core :as lib-be][metabase.lib.core :as lib]+ [metabase.lib.schema.id :as lib.schema.id][metabase.models.interface :as mi][metabase.models.serialization :as serdes][metabase.parameters.core :as parameters]+ [metabase.parameters.schema :as parameters.schema][metabase.public-sharing.core :as public-sharing][metabase.queries.models.query :as query]+ [metabase.queries.schema :as queries.schema][metabase.search.core :as search][metabase.util :as u][metabase.util.i18n :refer [tru]][metabase.util.malli :as mu]+ [metabase.util.malli.schema :as ms][methodical.core :as methodical][toucan2.core :as t2][toucan2.tools.hydrate :as t2.hydrate]))@@ -56,8 +60,8 @@(def ^:private transform-action-parameters"Like `parameters/transform-parameters`, but normalizes against `::actions.schema/action.parameters`: an implicitaction's parameters carry the annotations [[implicit-action-parameters]] computes them with."- {:in (comp mi/json-in actions.schema/normalize-parameters)- :out (comp (mi/catch-normalization-exceptions actions.schema/normalize-parameters)+ {:in (comp mi/json-in #(lib/normalize ::actions.schema/action.parameters %))+ :out (comp (mi/catch-normalization-exceptions #(lib/normalize ::actions.schema/action.parameters %))mi/json-out-with-keywordization)})(t2/deftransforms :model/Action@@ -76,9 +80,9 @@(def ^:private transform-json-with-nested-parameters{:in (comp mi/json-in(fn [template]- (u/update-if-exists template :parameters parameters/normalize-parameters)))+ (u/update-if-exists template :parameters #(lib/normalize ::parameters.schema/parameters %)))):out (comp (fn [template]- (u/update-if-exists template :parameters (mi/catch-normalization-exceptions parameters/normalize-parameters)))+ (u/update-if-exists template :parameters (mi/catch-normalization-exceptions #(lib/normalize ::parameters.schema/parameters %))))mi/json-out-with-keywordization)})(t2/deftransforms :model/HTTPAction@@ -161,7 +165,7 @@(mu/defn insert! :- ::actions.schema/id"Inserts an Action and related type table. Returns the action id."- [action-data :- :map]+ [action-data :- ::actions.schema/action.for-insert](insert*! (lib/normalize ::actions.schema/action.for-insert action-data)))(mu/defn- update*!@@ -196,8 +200,7 @@(mu/defn update!"Updates an Action and the related type table.Deletes the old type table row if the type has changed."- [updates :- [:map- [:id ::actions.schema/id]]+ [updates :- ::actions.schema/action.for-updateexisting-action :- ::actions.schema/action](let [updates (merge (select-keys existing-action [:type]) updates)] ; in case the updates do not include it.(update*! (lib/normalize ::actions.schema/action.for-update updates) existing-action)))@@ -430,14 +433,16 @@model for implicit actions.Pass in known-models to save a second Card lookup."- [known-models & options]+ [known-models :- [:maybe [:sequential ::queries.schema/card]]+ & options :- [:* [:or :nil :keyword ms/PositiveInt :string :boolean]]](enrich-actions-with-implicit-params known-models (apply select-actions-without-implicit-params options)))(mu/defn select-actions-for-ids :- [:maybe [:sequential ::actions.schema/action]]"Find the Actions whose `:id` is in `action-ids`, filling in implicit parameters as [[select-actions]] does.Pass in known-models to save a second Card lookup."- [known-models action-ids]+ [known-models :- [:maybe [:sequential ::queries.schema/card]]+ action-ids :- [:sequential ::lib.schema.id/action]](enrich-actions-with-implicit-params known-models (normalize-actions-by-type (actions.db/actions-with-ids action-ids))))(mu/defn select-actions-for-models :- [:maybe [:sequential ::actions.schema/action]]@@ -445,7 +450,8 @@[[select-actions]] does.Pass in known-models to save a second Card lookup."- [known-models model-ids]+ [known-models :- [:maybe [:sequential ::queries.schema/card]]+ model-ids :- [:sequential ms/PositiveInt]](enrich-actions-with-implicit-params known-models (normalize-actions-by-type (actions.db/unarchived-actions-for-models model-ids))))(mu/defn select-actions-non-http-for-models :- [:maybe [:sequential ::actions.schema/action]]@@ -453,13 +459,14 @@[[select-actions]] does.Pass in known-models to save a second Card lookup."- [known-models model-ids]+ [known-models :- [:maybe [:sequential ::queries.schema/card]]+ model-ids :- [:set ms/PositiveInt]](enrich-actions-with-implicit-params known-models (normalize-actions-by-type (actions.db/unarchived-non-http-actions-for-models model-ids))))(mu/defn select-action :- [:maybe ::actions.schema/action]"Selects an Action and fills in the subtype data and implicit parameters.`options` is interpreted by [[select-actions-matching-options]]."- [& options]+ [& options :- [:* [:or :nil :keyword ms/PositiveInt :string :boolean]]];; TODO -- it's dumb that we're selecting all matches rather than a single one above, limiting like this should;; never be done server-side. I don't have time to fix this right now. -- Cam(first (apply select-actions nil options)))diff --git a/src/metabase/actions/schema.clj b/src/metabase/actions/schema.cljindex a2be7f1bb31..ecbcccefcf3 100644--- a/src/metabase/actions/schema.clj+++ b/src/metabase/actions/schema.clj@@ -1,8 +1,9 @@(ns metabase.actions.schema(:require[metabase.actions.http-action :as http-action]+ [metabase.actions.types :as actions.types][metabase.lib-be.schema :as lib-be.schema]- [metabase.lib.core :as lib]+ [metabase.lib.schema.actions :as lib.schema.actions][metabase.lib.schema.common :as lib.schema.common][metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.parameter :as lib.schema.parameter]@@ -152,13 +153,6 @@(mr/def ::action.parameters[:sequential [:ref ::action.parameter]])-(mu/defn normalize-parameters :- ::action.parameters- "Normalize an Action's `:parameters` coming out of the application database or in via an API request. Like- [[metabase.parameters.schema/normalize-parameters]], but keeps the annotations an implicit action's parameters- carry."- [parameters]- (lib/normalize ::action.parameters parameters))-(mu/defn- action-schema [schema-type :- [:enum :select :update :insert]];; `required-for-insert` = you have to specify this when you insert a row;;@@ -216,11 +210,9 @@(mr/def ::httpaction"A HTTPAction as selected from the app DB: every column of `:http_action`."- [:map {:closed true}- [:action_id ::lib.schema.id/action]- [:template ::http-action.template]- [:response_handle [:maybe :string]]- [:error_handle [:maybe :string]]])+ [:merge+ ::httpaction.update+ [:map {:closed true}]])(mr/def ::httpaction.update"What an update (or insert) of a HTTPAction accepts: every column of `:http_action` except `id`, all optional."@@ -244,7 +236,7 @@(mr/def ::query-action.dataset-query"The `:dataset_query` column of a QueryAction, decoded."- :map)+ ::lib-be.schema/maybe-legacy-or-empty-query)(mr/def ::query-action.row"A QueryAction as selected from the app DB: every column of `:query_action`."@@ -261,3 +253,22 @@[:database_id {:optional true} [:maybe ::lib.schema.id/database]][:dataset_query {:optional true} [:maybe ::query-action.dataset-query]][:legacy_query {:optional true} [:maybe :string]]])++(mr/def ::execution.row-diff+ "One effect recorded against the `:effects` key of [[::execution-context]]: the before/after state of a row a+ perform-action!* method modified."+ [:map {:closed true}+ [:table-id ::lib.schema.id/table]+ [:db-id ::lib.schema.id/database]+ [:before [:maybe ::lib.schema.actions/row]]+ [:after [:maybe ::lib.schema.actions/row]]])++(mr/def ::execution-context+ "The `context` map threaded through `metabase.actions.actions/perform-action!*` and its driver implementations."+ [:map {:closed true}+ [:user-id {:optional true} [:maybe ms/PositiveInt]]+ [:scope {:optional true} [:maybe ::actions.types/scope.hydrated]]+ [:driver {:optional true} [:maybe :keyword]]+ [:invocation-id {:optional true} [:maybe :string]]+ [:invocation-stack {:optional true} [:maybe [:sequential [:tuple qualified-keyword? :string]]]]+ [:effects {:optional true} [:maybe [:sequential [:tuple qualified-keyword? ::execution.row-diff]]]]])diff --git a/src/metabase/actions/types.clj b/src/metabase/actions/types.cljindex 7a60ea04160..52bc1241bb7 100644--- a/src/metabase/actions/types.clj+++ b/src/metabase/actions/types.clj@@ -84,4 +84,11 @@[:dashcard-id ms/PositiveInt][:dashboard-id ::fk-or-missing-id][:collection-id ::fk-or-missing-id]]]- [:unknown any?]])+ [:unknown+ [:map {:closed true}+ [:type [:enum :unknown]]+ [:unknown {:optional true} [:enum :model-action]]+ [:card-id {:optional true} ms/PositiveInt]+ [:webhook-id {:optional true} ms/PositiveInt]+ [:collection-id {:optional true} ::fk-or-missing-id]+ [:database-id {:optional true} ::fk-or-missing-id]]]])diff --git a/src/metabase/activity_feed/db.clj b/src/metabase/activity_feed/db.cljindex ceec800c178..a2a96071872 100644--- a/src/metabase/activity_feed/db.clj+++ b/src/metabase/activity_feed/db.clj@@ -118,7 +118,7 @@(mu/defn table"The Table with `id`, or nil."- [id :- ::lib.schema.id/table]+ [id :- [:maybe ::lib.schema.id/table]](t2/select-one :model/Table :id id {:from [(warehouse-schema-overlay/table-query)]}))(mu/defn collectiondiff --git a/src/metabase/activity_feed/models/recent_views.clj b/src/metabase/activity_feed/models/recent_views.cljindex c16aa4aac1f..f406672fe05 100644--- a/src/metabase/activity_feed/models/recent_views.clj+++ b/src/metabase/activity_feed/models/recent_views.clj@@ -508,12 +508,14 @@Returns a sequence of [[Item]]s. The reason this isn't a `mu/defn`, is that error-avoider validates each Item in thesequence, so there's no need to do it twice."- ([user-id] (get-recents user-id [:views :selections]))- ([user-id context :- [:sequential [:enum :views :selections]]]+ ([user-id :- [:maybe ms/PositiveInt]]+ (get-recents user-id [:views :selections]))+ ([user-id :- [:maybe ms/PositiveInt]+ context :- [:sequential [:enum :views :selections]]](get-recents user-id context {}))- ([user-id+ ([user-id :- [:maybe ms/PositiveInt]context :- [:sequential [:enum :views :selections]]- options :- [:map+ options :- [:map {:closed true}[:include-metadata? {:optional true} [:maybe :boolean]][:models {:optional true} [:maybe [:sequential (into [:enum] rv-models)]]]]](let [models (:models options)diff --git a/src/metabase/activity_feed/schema.clj b/src/metabase/activity_feed/schema.cljindex 56fe2425183..57105c3f080 100644--- a/src/metabase/activity_feed/schema.clj+++ b/src/metabase/activity_feed/schema.clj@@ -7,13 +7,10 @@(mr/def ::recent-views"A RecentViews as selected from the app DB: every column of `:recent_views`."- [:map {:closed true}- [:id ms/PositiveInt]- [:user_id ::lib.schema.id/user]- [:model [:or :keyword :string]]- [:model_id [:maybe :int]]- [:timestamp ms/TemporalInstant]- [:context [:or :keyword :string]]])+ [:merge+ ::recent-views.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::recent-views.update"What an update (or insert) of a RecentViews accepts: every column of `:recent_views` except `id`, all optional."diff --git a/src/metabase/agent_api/api.clj b/src/metabase/agent_api/api.cljindex 269d04c2662..68ef06b478f 100644--- a/src/metabase/agent_api/api.clj+++ b/src/metabase/agent_api/api.clj@@ -862,7 +862,7 @@;; Mirror REST's `check-allowed-to-modify-query`: swapping the dataset_query requires data perms;; to run the *new* query, otherwise a user with collection write on a card can repoint it at data;; they cannot query. `queries/update-card!` does NOT run this check itself, so we run it here.- (when (api/column-will-change? :dataset_query card-before-update card-updates)+ (when (api/column-will-change? (:dataset_query card-before-update) (get card-updates :dataset_query ::api/not-provided))(query-perms/check-run-permissions-for-query (:dataset_query card-updates));; Reject cycles. `lib/check-card-overwrite` throws if the new query references this card;; transitively. Mirror REST's wrapping that promotes it to HTTP 400 instead of a 500.diff --git a/src/metabase/agent_api/schema.clj b/src/metabase/agent_api/schema.cljindex 9fea8cc95cb..37bc958ec56 100644--- a/src/metabase/agent_api/schema.clj+++ b/src/metabase/agent_api/schema.clj@@ -7,17 +7,10 @@(mr/def ::agent-api-call-log"A AgentApiCallLog as selected from the app DB: every column of `:agent_api_call_log`."- [:map {:closed true}- [:id ms/PositiveInt]- [:created_at ms/TemporalInstant]- [:user_id [:maybe ::lib.schema.id/user]]- [:tenant_id [:maybe ms/PositiveInt]]- [:client_name [:maybe :string]]- [:operation [:maybe :string]]- [:status [:maybe [:or :keyword :string]]]- [:duration_ms [:maybe :int]]- [:ip_address [:maybe :string]]- [:error_message [:maybe :string]]])+ [:merge+ ::agent-api-call-log.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::agent-api-call-log.update"What an update (or insert) of a AgentApiCallLog accepts: every column of `:agent_api_call_log` except `id`, all optional."diff --git a/src/metabase/analytics/llm_token_usage.clj b/src/metabase/analytics/llm_token_usage.cljindex 74eb619faf0..471c0b803db 100644--- a/src/metabase/analytics/llm_token_usage.clj+++ b/src/metabase/analytics/llm_token_usage.clj@@ -1,6 +1,7 @@(ns metabase.analytics.llm-token-usage"LLM token usage tracking for Snowplow and Prometheus."(:require+ [malli.util :as mut][metabase.analytics-interface.core :as analytics][metabase.analytics.event :as analytics.event][metabase.analytics.util :as analytics.util]@@ -9,8 +10,13 @@(set! *warn-on-reflection* true)+(def ^:private snowplow-arg-keys+ [:request-id :model-id :total-tokens :prompt-tokens :completion-tokens :estimated-costs-usd+ :cache-creation-tokens :cache-read-tokens :user-id :duration-ms :source :tag :session-id :profile+ :hashed-metabase-license-token])+(def ^:private SnowplowArgs- [:map+ [:map {:closed true}[:request-id :string][:model-id :string][:total-tokens ms/IntGreaterThanOrEqualToZero]@@ -53,8 +59,11 @@:profile profile}user-id))+(def ^:private prometheus-arg-keys+ [:model-id :tag :prompt-tokens :completion-tokens :cache-creation-tokens :cache-read-tokens])+(def ^:private PrometheusArgs- [:map+ [:map {:closed true}[:model-id :string][:tag :string][:prompt-tokens ms/IntGreaterThanOrEqualToZero]@@ -76,6 +85,27 @@(analytics/inc! :metabase-metabot/llm-cache-read-tokens labels cache-read-tokens))(analytics/observe! :metabase-metabot/llm-tokens-per-call labels (+ prompt-tokens completion-tokens))))+(def ^:private TrackTokenUsageGates+ "The `:snowplow`/`:prometheus` gate keys of [[track-token-usage!]]."+ [:map {:closed true}+ [:snowplow :boolean]+ [:prometheus :boolean]])++(def ^:private UntrackedSnowplowArgs+ "The Snowplow-only keys a Prometheus-only [[track-token-usage!]] call may still carry, possibly nil."+ (mut/merge (mut/optional-keys SnowplowArgs)+ [:map {:closed true}+ [:request-id {:optional true} [:maybe :string]]+ [:total-tokens {:optional true} [:maybe ms/IntGreaterThanOrEqualToZero]]+ [:estimated-costs-usd {:optional true} [:maybe number?]]]))++(def ^:private TrackTokenUsageArgs+ "The gate keys plus the keys of whichever of [[track-snowplow!]] and [[track-prometheus!]] the gates enable."+ [:multi {:dispatch (juxt :snowplow :prometheus)}+ [[true false] (mut/merge TrackTokenUsageGates SnowplowArgs)]+ [[false true] (mut/merge (mut/merge TrackTokenUsageGates UntrackedSnowplowArgs) PrometheusArgs)]+ [[true true] (mut/merge (mut/merge TrackTokenUsageGates SnowplowArgs) PrometheusArgs)]])+(mu/defn track-token-usage!"Convenience wrapper that fires Snowplow and/or Prometheus token tracking.@@ -83,17 +113,11 @@- `:snowplow` (required boolean) — pass `false` to suppress Snowplow- `:prometheus` (required boolean) — pass `false` to suppress Prometheus"[{:keys [snowplow prometheus] :as opts}- :- [:merge- [:map- [:snowplow [:boolean {:default true}]]- [:prometheus [:boolean {:default true}]]]- [:multi {:dispatch (juxt :snowplow :prometheus)}- [[true false] SnowplowArgs]- [[false true] PrometheusArgs]- [[true true] [:merge SnowplowArgs PrometheusArgs]]- [[false false] [:fn {:error/message "at least one of :snowplow or :prometheus must be true"}- (constantly false)]]]]]+ :- [:and+ [:fn {:error/message "at least one of :snowplow or :prometheus must be true"}+ (fn [{:keys [snowplow prometheus]}] (or snowplow prometheus))]+ TrackTokenUsageArgs]](when snowplow- (track-snowplow! opts))+ (track-snowplow! (select-keys opts snowplow-arg-keys)))(when prometheus- (track-prometheus! opts)))+ (track-prometheus! (select-keys opts prometheus-arg-keys))))diff --git a/src/metabase/analytics/metaplow.clj b/src/metabase/analytics/metaplow.cljindex 784f6e817d8..a1b02d07c58 100644--- a/src/metabase/analytics/metaplow.clj+++ b/src/metabase/analytics/metaplow.clj@@ -15,11 +15,13 @@[clojure.walk :as walk][metabase.analytics-interface.core :as analytics][metabase.analytics.settings :as analytics.settings]+ [metabase.analytics.snowplow :as snowplow][metabase.premium-features.core :as premium-features][metabase.util :as u][metabase.util.json :as json][metabase.util.log :as log][metabase.util.malli :as mu]+ [metabase.util.malli.schema :as ms][metabase.util.retry :as retry][metabase.version.core :as version]))@@ -162,9 +164,9 @@(mu/defn track-event! :- :boolean"Send a single analytics event to the Metaplow collector. Returns true when the event was enqueued, false whenMetaplow tracking is disabled or the queue is full."- ([schema data]+ ([schema :- snowplow/SnowplowSchema data :- snowplow/SnowplowEventData](track-event! schema data nil))- ([schema :- :keyword data _user-id]+ ([schema :- snowplow/SnowplowSchema data :- snowplow/SnowplowEventData _user-id :- [:maybe ms/PositiveInt]](boolean(when (analytics.settings/metaplow-tracking-enabled)(trydiff --git a/src/metabase/analytics/sdk.clj b/src/metabase/analytics/sdk.cljindex dde9487ac3a..93347bc1cc0 100644--- a/src/metabase/analytics/sdk.clj+++ b/src/metabase/analytics/sdk.clj@@ -16,7 +16,8 @@[metabase.request.current :as request.current][metabase.request.user-agent :as request.user-agent][metabase.util.log :as log]- [metabase.util.malli :as mu])+ [metabase.util.malli :as mu]+ [metabase.util.malli.schema :as ms])(:import(java.net URI)))@@ -119,10 +120,55 @@:sanitized_user_agent (request.user-agent/describe-user-agent (get-in request [:headers "user-agent"])):ip_address (request.current/ip-address request)})))+(def ^:private sdk-info-row+ [:map {:closed true}+ [:user_id {:optional true} [:maybe :int]]+ [:model {:optional true} [:maybe [:or :keyword :string]]]+ [:model_id {:optional true} [:maybe :int]]+ [:timestamp {:optional true} [:maybe [:or ms/TemporalInstant :keyword]]]+ [:metadata {:optional true} [:maybe [:map {:closed true}]]]+ [:has_access {:optional true} [:maybe :boolean]]+ [:hash {:optional true} [:maybe [:or bytes? :string]]]+ [:started_at {:optional true} [:maybe ms/TemporalInstant]]+ [:running_time {:optional true} [:maybe :int]]+ [:result_rows {:optional true} [:maybe :int]]+ [:native {:optional true} [:maybe :boolean]]+ [:context {:optional true} [:maybe [:or :keyword :string]]]+ [:error {:optional true} [:maybe :string]]+ [:executor_id {:optional true} [:maybe :int]]+ [:card_id {:optional true} [:maybe :int]]+ [:dashboard_id {:optional true} [:maybe :int]]+ [:pulse_id {:optional true} [:maybe :int]]+ [:database_id {:optional true} [:maybe :int]]+ [:cache_hit {:optional true} [:maybe :boolean]]+ [:action_id {:optional true} [:maybe :int]]+ [:is_sandboxed {:optional true} [:maybe :boolean]]+ [:cache_hash {:optional true} [:maybe [:or bytes? :string]]]+ [:embedding_client {:optional true} [:maybe :string]]+ [:embedding_sdk_version {:optional true} [:maybe :string]]+ [:parameterized {:optional true} [:maybe :boolean]]+ [:transform_id {:optional true} [:maybe :int]]+ [:lens_id {:optional true} [:maybe :string]]+ [:lens_params {:optional true} [:maybe [:map {:closed true} [:join_step {:optional true} [:maybe :int]]]]]+ [:auth_method {:optional true} [:maybe [:or :keyword :string]]]+ [:tenant_id {:optional true} [:maybe :int]]+ [:is_impersonated {:optional true} [:maybe :boolean]]+ [:is_db_routed {:optional true} [:maybe :boolean]]+ [:parameters {:optional true} [:maybe :string]]+ [:embedding_hostname {:optional true} [:maybe :string]]+ [:embedding_path {:optional true} [:maybe :string]]+ [:user_agent {:optional true} [:maybe :string]]+ [:ip_address {:optional true} [:maybe :string]]+ [:sanitized_user_agent {:optional true} [:maybe :string]]+ [:embedding_route {:optional true} [:maybe :string]]+ [:metabase_version {:optional true} [:maybe :string]]+ [:embedding_client_identifier {:optional true} [:maybe :string]]+ [:start_time_millis {:optional true} [:maybe :int]]])+(mu/defn include-sdk-info :- :map"Adds the currently bound, or existing `*client*` and `*version*` to the given map, which is usually a row goinginto the `view_log` or `query_execution` table. Falls back to the original value."- [m :- :map]+ [m :- sdk-info-row](-> m(update :embedding_client (fn [client] (or *client* client)))(update :embedding_client_identifier (fn [identifier] (or *client-identifier* identifier)))diff --git a/src/metabase/analytics/snowplow.clj b/src/metabase/analytics/snowplow.cljindex fe4e5c41897..5db9801e0f3 100644--- a/src/metabase/analytics/snowplow.clj+++ b/src/metabase/analytics/snowplow.clj@@ -8,6 +8,7 @@[metabase.premium-features.core :as premium-features][metabase.util.log :as log][metabase.util.malli :as mu]+ [metabase.util.malli.schema :as ms][metabase.version.core :as version][toucan2.core :as t2])(:import@@ -57,10 +58,189 @@:snowplow/ai_service_event "1-0-0":snowplow/data_complexity "1-0-0"})-(def ^:private SnowplowSchema+(def SnowplowSchema"Malli enum for valid Snowplow schemas"(into [:enum] (keys schema->version)))+(def ^:private no-payload-event-data+ "Shape for `SnowplowSchema` values with no caller in this codebase today."+ [:map {:closed true}+ [:event {:optional true} :keyword]])++(def ^:private account-event-data+ [:map {:closed true}+ [:event {:optional true} :keyword]])++(def ^:private invite-event-data+ [:map {:closed true}+ [:event {:optional true} :keyword]+ [:invited-user-id {:optional true} ms/PositiveInt]+ [:source {:optional true} [:or :keyword :string]]])++(def ^:private dashboard-event-data+ [:map {:closed true}+ [:event {:optional true} :keyword]+ [:dashboard-id {:optional true} ms/PositiveInt]+ [:question-id {:optional true} ms/PositiveInt]+ [:num-tabs {:optional true} :int]+ [:total-num-tabs {:optional true} :int]])++(def ^:private database-event-data+ [:map {:closed true}+ [:event {:optional true} :keyword]+ [:database {:optional true} [:or :keyword :string]]+ [:database-id {:optional true} ms/PositiveInt]+ [:source {:optional true} [:or :keyword :string]]+ [:dbms-version {:optional true} [:maybe :string]]+ [:dbms_version {:optional true} :string]])++(def ^:private simple-event-data+ [:map {:closed true}+ [:event {:optional true} [:or :string :keyword]]+ [:event_detail {:optional true} [:maybe :string]]+ [:event-detail {:optional true} [:maybe :string]]+ [:target_id {:optional true} ms/PositiveInt]+ [:duration_ms {:optional true} [:maybe :int]]+ [:result {:optional true} :string]+ [:triggered_from {:optional true} :string]])++(def ^:private timeline-event-data+ [:map {:closed true}+ [:event {:optional true} :keyword]+ [:time_matters {:optional true} [:maybe :boolean]]+ [:collection_id {:optional true} [:maybe ms/PositiveInt]]+ [:source {:optional true} :string]+ [:question_id {:optional true} ms/PositiveInt]])++(def ^:private action-event-data+ [:map {:closed true}+ [:event {:optional true} :keyword]+ [:source {:optional true} :keyword]+ [:type {:optional true} :keyword]+ [:action_id {:optional true} ms/PositiveInt]+ [:num_parameters {:optional true} :int]])++(def ^:private embed-share-event-data+ [:map {:closed true}+ [:event {:optional true} :keyword]+ [:embedding-app-origin-set {:optional true} :boolean]+ [:number-embedded-questions {:optional true} :int]+ [:number-embedded-dashboards {:optional true} :int]])++(def ^:private model-event-data+ [:map {:closed true}+ [:event {:optional true} :keyword]+ [:model-id {:optional true} ms/PositiveInt]])++(def ^:private csvupload-event-data+ [:map {:closed true}+ [:event {:optional true} :keyword]+ [:num-rows {:optional true} :int]+ [:num-columns {:optional true} :int]+ [:generated-columns {:optional true} :int]+ [:size-mb {:optional true} number?]+ [:upload-seconds {:optional true} number?]+ [:model-id {:optional true} ms/PositiveInt]])++(def ^:private token-usage-event-data+ [:map {:closed true}+ [:request-id {:optional true} [:maybe :string]]+ [:model-id {:optional true} [:maybe :string]]+ [:total-tokens {:optional true} [:maybe ms/IntGreaterThanOrEqualToZero]]+ [:prompt-tokens {:optional true} [:maybe ms/IntGreaterThanOrEqualToZero]]+ [:completion-tokens {:optional true} [:maybe ms/IntGreaterThanOrEqualToZero]]+ [:estimated-costs-usd {:optional true} [:maybe number?]]+ [:cache-creation-tokens {:optional true} [:maybe ms/IntGreaterThanOrEqualToZero]]+ [:cache-read-tokens {:optional true} [:maybe ms/IntGreaterThanOrEqualToZero]]+ [:user-id {:optional true} [:maybe :int]]+ [:duration-ms {:optional true} [:maybe ms/IntGreaterThanOrEqualToZero]]+ [:source {:optional true} [:maybe :string]]+ [:tag {:optional true} [:maybe :string]]+ [:session-id {:optional true} [:maybe :string]]+ [:profile {:optional true} [:maybe :string]]+ [:hashed-metabase-license-token {:optional true} [:maybe :string]]])++(def ^:private serialization-event-data+ [:map {:closed true}+ [:event {:optional true} :keyword]+ [:direction {:optional true} :string]+ [:source {:optional true} :string]+ [:duration_ms {:optional true} :int]+ [:models {:optional true} :string]+ [:count {:optional true} :int]+ [:error_count {:optional true} :int]+ [:success {:optional true} :boolean]+ [:error_message {:optional true} [:maybe :string]]+ [:collection {:optional true} :string]+ [:all_collections {:optional true} :boolean]+ [:data_model {:optional true} :boolean]+ [:settings {:optional true} :boolean]+ [:field_values {:optional true} [:maybe :boolean]]+ [:secrets {:optional true} :boolean]])++(def ^:private cleanup-event-data+ [:map {:closed true}+ [:event {:optional true} :keyword]+ [:collection_id {:optional true} [:maybe ms/PositiveInt]]+ [:total_stale_items_found {:optional true} :int]+ [:cutoff_date {:optional true} :string]])++(def ^:private data-complexity-event-data+ [:map {:closed true}+ [:event {:optional true} :keyword]+ [:batch_id {:optional true} :string]+ [:formula_version {:optional true} :int]+ [:parameters {:optional true}+ [:map {:closed true}+ ["synonym_threshold" {:optional true} [:maybe number?]]+ ["weights" {:optional true} [:maybe ms/OpaqueJSONObject]]+ ["embedding_model" {:optional true} [:maybe ms/OpaqueJSONObject]]+ ["text_variant" {:optional true} [:maybe :string]]]]+ [:catalog {:optional true} :keyword]+ [:key {:optional true} :string]+ [:score {:optional true} number?]+ [:measurement {:optional true} number?]+ [:error {:optional true} :string]])++(def ^:private ai-service-event-data+ [:map {:closed true}+ [:hashed-metabase-license-token {:optional true} [:maybe :string]]+ [:request-id {:optional true} [:maybe :string]]+ [:source {:optional true} [:maybe :string]]+ [:event {:optional true} :string]+ [:user-id {:optional true} [:maybe ms/PositiveInt]]+ [:session-id {:optional true} [:maybe :string]]+ [:profile {:optional true} [:maybe :string]]+ [:duration-ms {:optional true} [:maybe :int]]+ [:result {:optional true} :string]+ [:event-details {:optional true}+ [:map {:closed true}+ ["tool_name" {:optional true} [:maybe :string]]+ ["step" {:optional true} :int]]]])++(def SnowplowEventData+ "Closed shape of the `data` payload accepted by [[track-event!]], as a union of the shapes each `SnowplowSchema`+ value actually carries at its call sites. `:snowplow/instance_stats` is an opaque, deeply-nested telemetry blob+ assembled from many stats sources and forwarded to Snowplow without being read by key."+ [:or+ account-event-data+ invite-event-data+ dashboard-event-data+ database-event-data+ simple-event-data+ timeline-event-data+ action-event-data+ embed-share-event-data+ model-event-data+ csvupload-event-data+ token-usage-event-data+ serialization-event-data+ cleanup-event-data+ data-complexity-event-data+ ai-service-event-data+ ms/OpaqueJSONObject+ no-payload-event-data])+(defn- tracker-config[](TrackerConfiguration. "sp" "metabase"))@@ -149,10 +329,10 @@"Send a single analytics event to the Snowplow collector, if tracking is enabled for this MB instance and a collectoris available. Returns true when the event was actually handed to the tracker; false when tracking is disabled oremission threw — callers that need to gate durable side-effects on real delivery can check the return value."- ([schema :- SnowplowSchema data]+ ([schema :- SnowplowSchema data :- SnowplowEventData](track-event! schema data api/*current-user-id*))- ([schema :- SnowplowSchema data user-id]+ ([schema :- SnowplowSchema data :- SnowplowEventData user-id :- [:maybe ms/PositiveInt]](boolean(when (analytics.settings/snowplow-enabled)(trydiff --git a/src/metabase/analytics/stats.clj b/src/metabase/analytics/stats.cljindex acf162fe496..5664f01aaa3 100644--- a/src/metabase/analytics/stats.clj+++ b/src/metabase/analytics/stats.clj@@ -27,6 +27,7 @@[metabase.util.json :as json][metabase.util.log :as log][metabase.util.malli :as mu]+ [metabase.util.malli.schema :as ms][metabase.version.core :as version]))(set! *warn-on-reflection* true)@@ -539,6 +540,15 @@{:group (str qe-group) :value (get executions qe-group)})[:interactive_embed :internal :public_link :sdk_embed :simple_embed :static_embed]))+(def ^:private QueryExecutionCounts+ [:map {:closed true}+ [:sdk_embed :int]+ [:interactive_embed :int]+ [:static_embed :int]+ [:public_link :int]+ [:simple_embed :int]+ [:internal :int]])+(mu/defn- snowplow-grouped-metrics:- [:sequential[:map@@ -548,7 +558,12 @@[{:keys [eid-translations-24hquery-executionsquery-executions-24h]- :as _snowplow-grouped-metric-info}]+ :as _snowplow-grouped-metric-info}+ :- [:map {:closed true}+ [:eid-translations-24h [:map {:closed true}+ [:ok :int] [:not-found :int] [:invalid-format :int] [:total :int]]]+ [:query-executions QueryExecutionCounts]+ [:query-executions-24h QueryExecutionCounts]]](deep-string-keywords[{:name :query_executions_by_source:values (get-query-exeuction-counts query-executions)@@ -582,16 +597,159 @@:scim_users_last_24h (analytics.db/new-scim-user-count-since one-day-ago)}(transform-metrics))))+(def ^:private string-keyed-int-histogram+ [:map-of :string :int])++(def ^:private optional-count-map+ "Non-negative integer counts keyed by whatever subset of these metric names a given call site fills in."+ [:map {:closed true}+ [:total {:optional true} :int]+ [:native {:optional true} :int]+ [:gui {:optional true} :int]+ [:is_dashboard_question {:optional true} :int]+ [:with_params {:optional true} :int]+ [:with_enabled_params {:optional true} :int]+ [:with_locked_params {:optional true} :int]+ [:with_disabled_params {:optional true} :int]+ [:active {:optional true} :int]+ [:admin {:optional true} :int]+ [:logged_in {:optional true} :int]+ [:sso {:optional true} :int]+ [:archived {:optional true} :int]+ [:analyzed {:optional true} :int]])++(def ^:private LegacyAnonymousUsageStats+ "The shape returned by [[legacy-anonymous-usage-stats]]: instance metadata plus a `:stats` map of per-domain+ metric groups."+ [:map {:closed true}+ [:version :string]+ [:running_on :keyword]+ [:startup_time_millis :int]+ [:application_database [:maybe :string]]+ [:check_for_updates :boolean]+ [:report_timezone [:maybe :string]]+ [:friendly_names :boolean]+ [:email_configured :boolean]+ [:slack_configured :boolean]+ [:sso_configured :boolean]+ [:instance_started :string]+ [:has_sample_data :boolean]+ [:enable_embedding :boolean]+ [:enable_embedding_sdk :boolean]+ [:enable_embedding_simple :boolean]+ [:enable_embedding_interactive :boolean]+ [:enable_embedding_static :boolean]+ [:enable_embedding_modular :boolean]+ [:embedding_app_origin_set :boolean]+ [:embedding_app_origin_sdk_set :boolean]+ [:embedding_app_origin_interactive_set [:maybe :string]]+ [:appearance_site_name :boolean]+ [:appearance_help_link :keyword]+ [:appearance_logo :boolean]+ [:appearance_favicon :boolean]+ [:appearance_loading_message :boolean]+ [:appearance_metabot_greeting :boolean]+ [:appearance_login_page_illustration :string]+ [:appearance_landing_page_illustration :string]+ [:appearance_no_data_illustration :string]+ [:appearance_no_object_illustration :string]+ [:appearance_ui_colors :boolean]+ [:appearance_chart_colors :boolean]+ [:appearance_show_mb_links :boolean]+ [:uuid :string]+ [:timestamp (ms/InstanceOfClass java.time.OffsetDateTime)]+ [:stats+ [:map {:closed true}+ [:cache [:map {:closed true}+ [:average_entry_size :int]+ [:num_queries_cached :string]+ [:num_queries_cached_unbinned :int]]]+ [:collection [:map {:closed true}+ [:collections :int]+ [:cards_in_collections :int]+ [:cards_not_in_collections :int]+ [:num_cards_per_collection string-keyed-int-histogram]]]+ [:dashboard [:map {:closed true}+ [:dashboards :int]+ [:with_params :int]+ [:num_dashs_per_user string-keyed-int-histogram]+ [:num_cards_per_dash string-keyed-int-histogram]+ [:num_dashs_per_card string-keyed-int-histogram]+ [:public optional-count-map]+ [:embedded optional-count-map]]]+ [:database [:map {:closed true}+ [:databases optional-count-map]+ [:dbms_versions string-keyed-int-histogram]]]+ [:execution [:map {:closed true}+ [:executions :int]+ [:by_status string-keyed-int-histogram]+ [:num_per_user string-keyed-int-histogram]+ [:num_by_latency string-keyed-int-histogram]]]+ [:field [:map {:closed true}+ [:fields :int]+ [:num_per_table string-keyed-int-histogram]]]+ [:group [:map {:closed true}+ [:groups :int]]]+ [:metric [:map {:closed true}+ [:metrics :int]]]+ [:pulse [:map {:closed true}+ [:pulses :int]+ [:with_table_cards :int]+ [:pulse_types string-keyed-int-histogram]+ [:pulse_schedules string-keyed-int-histogram]+ [:num_pulses_per_user string-keyed-int-histogram]+ [:num_pulses_per_card string-keyed-int-histogram]+ [:num_cards_per_pulses string-keyed-int-histogram]]]+ [:alert [:map {:closed true}+ [:alerts :int]+ [:with_table_cards :int]+ [:first_time_only :int]+ [:above_goal :int]+ [:alert_types string-keyed-int-histogram]+ [:num_alerts_per_user string-keyed-int-histogram]+ [:num_alerts_per_card string-keyed-int-histogram]+ [:num_cards_per_alerts string-keyed-int-histogram]]]+ [:question [:map {:closed true}+ [:questions optional-count-map]+ [:public optional-count-map]+ [:embedded optional-count-map]]]+ [:segment [:map {:closed true}+ [:segments :int]]]+ [:system [:map {:closed true}+ [:max_memory :int]+ [:processors :int]+ [:java_version [:maybe :string]]+ [:java_vm_specification_version [:maybe :string]]+ [:java_runtime_name [:maybe :string]]+ [:user_timezone [:maybe :string]]+ [:user_language [:maybe :string]]+ [:user_country [:maybe :string]]+ [:file_encoding [:maybe :string]]+ [:os_name [:maybe :string]]+ [:os_version [:maybe :string]]]]+ [:table [:map {:closed true}+ [:tables :int]+ [:num_per_database string-keyed-int-histogram]+ [:num_per_schema string-keyed-int-histogram]]]+ [:user [:map {:closed true}+ [:users optional-count-map]]]+ [:document [:map {:closed true}+ [:documents optional-count-map]]]+ [:library [:map {:closed true}+ [:library_data :int]+ [:library_metrics :int]]]]]])+(mu/defn- snowplow-metrics- [stats metric-info :- [:map- [:models :int]- [:new_embedded_dashboards :int]- [:new_users_last_24h :int]- [:pivot_tables :int]- [:query_executions_last_24h :int]- [:entity_id_translations_last_24h :int]- [:transforms :int]- [:transform_runs_last_24h :int]]]+ [stats :- LegacyAnonymousUsageStats+ metric-info :- [:map {:closed true}+ [:models :int]+ [:new_embedded_dashboards :int]+ [:new_users_last_24h :int]+ [:pivot_tables :int]+ [:query_executions_last_24h :int]+ [:entity_id_translations_last_24h :int]+ [:transforms :int]+ [:transform_runs_last_24h :int]]](mapv(fn [[k v tags]](assert (every? string? tags) "Tags must be strings in snowplow metrics.")diff --git a/src/metabase/analyze/classifiers/data_sensitivity.clj b/src/metabase/analyze/classifiers/data_sensitivity.cljindex 8259ef62dca..d5f3924dc1e 100644--- a/src/metabase/analyze/classifiers/data_sensitivity.clj+++ b/src/metabase/analyze/classifiers/data_sensitivity.clj@@ -243,16 +243,9 @@(some rule-table-tokens table-tokens))]category))))-(def ^:private Field- [:map- [:name :string]- [:base_type :keyword]- [:semantic_type {:optional true} [:maybe :keyword]]- [:fingerprint {:optional true} :any]])-(def ^:private TableContext[:maybe- [:map+ [:map {:closed true}[:name {:optional true} [:maybe :string]][:entity_type {:optional true} [:maybe :keyword]]]])@@ -260,7 +253,7 @@"Infer the `data_sensitivity` category of `field` from its name, base type, semantic type, and fingerprint, with`table-context` (`:name`, `:entity_type` of its Table) gating the weaker rules. Returns the highest-precedencematching category, or nil when no rule matches. Never returns `:PUBLIC`."- [{field-name :name, :keys [base_type semantic_type fingerprint]} :- Field+ [{field-name :name, :keys [base_type semantic_type fingerprint]} :- :metabase.warehouse-schema.schema/field{table-name :name, :keys [entity_type]} :- TableContext](let [tokens (name->tokens field-name)](->> (concat (token-matches tokens base_type)diff --git a/src/metabase/analyze/classifiers/name.clj b/src/metabase/analyze/classifiers/name.cljindex f101ec5958b..adc1438e3f4 100644--- a/src/metabase/analyze/classifiers/name.clj+++ b/src/metabase/analyze/classifiers/name.clj@@ -129,15 +129,21 @@semantic-type))pattern+base-types+semantic-type)))+(def ^:private MinimalFieldOrColumn+ "The subset of a Field/column map this classifier actually reads: name, base type, and (when already+ known) semantic type."+ [:map {:closed true}+ [:name {:optional true} [:maybe :string]]+ [:base_type {:optional true} [:maybe ms/FieldType]]+ [:semantic_type {:optional true} [:maybe ms/FieldSemanticOrRelationType]]])+(def ^:private FieldOrColumn- "Schema that allows a `:model/Field` or a column from a query resultset"- [:and- [:map- ;; Some DBs such as MSSQL can return columns with blank name- [:name :string]- [:base_type :keyword]- [:semantic_type {:optional true} [:maybe :keyword]]]- ::analyze.schema/qp-results-cased-map])+ "Schema that allows a `:model/Field`, a column from a query resultset, or a minimal name/type map (as+ hand-built test fixtures pass)."+ [:or+ ::analyze.schema/Field+ :metabase.legacy-mbql.schema/legacy-column-metadata+ MinimalFieldOrColumn])(mu/defn infer-semantic-type-by-name :- [:maybe :keyword]"Classifier that infers the semantic type of a `field` based on its name and base type."diff --git a/src/metabase/analyze/query_results.clj b/src/metabase/analyze/query_results.cljindex 1f11b960cf1..fd1b6ebef69 100644--- a/src/metabase/analyze/query_results.clj+++ b/src/metabase/analyze/query_results.clj@@ -30,7 +30,7 @@(mu/defn- maybe-infer-semantic-type :- ResultColumnMetadata"Infer the semantic type and add it to the result metadata. If the inferred semantic type is nil, don't override thesemantic type with a nil semantic type"- [col]+ [col :- ResultColumnMetadata](updatecol:semantic_type@@ -44,7 +44,7 @@(mu/defn- col->ResultColumnMetadata :- ResultColumnMetadata"Make sure a `column` as it comes back from a driver's initial results metadata matches the schema for valid resultscolumn metadata, adding placeholder values and removing nil keys."- [column]+ [column :- ResultColumnMetadata];; HACK - not sure why we don't have display_name yet in some cases(merge{:base_type :type/*@@ -55,7 +55,7 @@"A reducing function that calculates what is ultimately returned as `[:data :results_metadata]` in userland QPresults. `metadata` is the usual QP results metadata e.g. as received by an `rff`."{:arglists '([metadata])}- [{:keys [cols]}]+ [{:keys [cols]} :- ::query-processor.schema/metadata](let [cols (for [col cols](try(maybe-infer-semantic-type (col->ResultColumnMetadata col))diff --git a/src/metabase/api/common.clj b/src/metabase/api/common.cljindex d170fcda6e9..35542725b85 100644--- a/src/metabase/api/common.clj+++ b/src/metabase/api/common.clj@@ -445,21 +445,28 @@(check (not (and limit (not offset))) [400 (tru "When including a limit, an offset must also be included.")])(check (not (and offset (not limit))) [400 (tru "When including an offset, a limit must also be included.")]))+(def ^:private ColumnValue+ "One value of a column compared by `column-will-change?`."+ [:maybe [:or :string :int :boolean :keyword+ :metabase.queries.schema/card.dataset-query+ :metabase.queries.schema/card.result-metadata+ ms/EmbeddingParams]])+(mu/defn column-will-change? :- :boolean- "Helper for PATCH-style operations to see if a column is set to change when `object-updates` (i.e., the input to the- endpoint) is applied.+ "Helper for PATCH-style operations to see if a column is set to change when `after` (the column's value in the+ input to the endpoint, or `::not-provided` when the column is absent from that input) is applied.;; assuming we have a Collection 10, that is not currently archived...- (api/column-will-change? :archived (t2/select-one Collection :id 10) {:archived true}) ; -> true, because value will change+ (api/column-will-change? false true) ; -> true, because value will change- (api/column-will-change? :archived (t2/select-one Collection :id 10) {:archived false}) ; -> false, because value did not change+ (api/column-will-change? false false) ; -> false, because value did not change- (api/column-will-change? :archived (t2/select-one Collection :id 10) {}) ; -> false; value not specified in updates (request body)"- [k :- :keyword object-before-updates :- :map object-updates :- :map]+ (api/column-will-change? false ::not-provided) ; -> false; value not specified in updates (request body)"+ [before :- ColumnValue+ after :- [:or ColumnValue [:= ::not-provided]]](boolean- (and (contains? object-updates k)- (not= (get object-before-updates k)- (get object-updates k)))))+ (and (not= after ::not-provided)+ (not= before after))));;; ------------------------------------------ COLLECTION POSITION HELPER FNS ----------------------------------------@@ -494,14 +501,14 @@(def ^:private ModelWithPosition"Intended to cover Cards/Dashboards/Pulses, it only asserts collection id and position, allowing extra keys"- [:map+ [:map {:closed true}[:collection_id [:maybe ms/PositiveInt]][:collection_position [:maybe ms/PositiveInt]]])(def ^:private ModelWithOptionalPosition"Intended to cover Cards/Dashboards/Pulses updates. Collection id and position are optional, if they are notpresent, they didn't change. If they are present, they might have changed and we need to compare."- [:map+ [:map {:closed true}[:collection_id {:optional true} [:maybe ms/PositiveInt]][:collection_position {:optional true} [:maybe ms/PositiveInt]]])@@ -510,7 +517,7 @@impact to the collection position of that model instance. If so, executes updates to fix the collection positionthat goes with the change. The 2-arg version of this function is used for a new card/dashboard/pulse (i.e. notupdating an existing instance, but creating a new one)."- ([new-model-data :- ModelWithPosition]+ ([new-model-data :- ModelWithOptionalPosition](maybe-reconcile-collection-position! nil new-model-data))([{old-collection-id :collection_id, old-position :collection_position, :as _before-update} :- [:maybe ModelWithPosition]{new-collection-id :collection_id, new-position :collection_position, :as model-updates} :- ModelWithOptionalPosition]@@ -578,7 +585,7 @@"Sets `archived_directly` to `true` iff `:archived` is being set to `true`."[current-obj obj-updates](cond-> obj-updates- (column-will-change? :archived current-obj obj-updates)+ (column-will-change? (:archived current-obj) (get obj-updates :archived ::not-provided))(assoc :archived_directly (boolean (:archived obj-updates)));; This is a hack around a frontend issue. Apparently, the undo functionality depends on calculating a diff@@ -587,7 +594,7 @@;;;; Let's just say that if you're marking something as archived, we throw away any `collection_id` you passed in;; along with it.- (and (column-will-change? :archived current-obj obj-updates)+ (and (column-will-change? (:archived current-obj) (get obj-updates :archived ::not-provided))(:archived obj-updates))(dissoc :collection_id)))@@ -603,9 +610,10 @@and an `:id`. The `f` function is called like `(f model all-items-with-that-model)` and should return a collectionof maps. `:id` is the only required key for these maps, and order *does not matter* - `present-items` is responsiblefor reordering items the way they were."- [f items :- [:sequential [:map- [:id ms/PositiveInt]- [:model :keyword]]]]+ [f :- ifn?+ items :- [:sequential [:map {:closed true}+ [:id ms/PositiveInt]+ [:model :keyword]]]](let [id+model->order (into {} (map-indexed (fn [i row] [[(:id row) (:model row)] i]) items))](->> items(group-by :model)diff --git a/src/metabase/api/db.clj b/src/metabase/api/db.cljindex 89deb15be12..3f5255848d7 100644--- a/src/metabase/api/db.clj+++ b/src/metabase/api/db.clj@@ -4,17 +4,23 @@(:require[metabase.lib.schema.id :as lib.schema.id][metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms][toucan2.core :as t2]))+(mr/def ::condition-value+ "A key or value of a Toucan 2 key-value condition: a scalar, or a HoneySQL comparison clause like `[:not= nil]`."+ [:or :keyword :string :boolean ms/PositiveInt+ [:cat :keyword [:* [:maybe [:or :string :boolean ms/PositiveInt :keyword]]]]])+(mu/defn entity-exists?"Whether a row of `entity` matching the key-value `conditions` exists."- [entity :- :keyword & conditions :- [:* :some]]+ [entity :- :keyword & conditions :- [:* [:maybe ::condition-value]]](apply t2/exists? entity conditions))(mu/defn entity-by-id"The `entity` row with `id` also matching the key-value `conditions`, or nil."- [entity :- :keyword id :- [:or ms/PositiveInt :string] & conditions :- [:* :some]]+ [entity :- :keyword id :- [:maybe [:or ms/PositiveInt :string]] & conditions :- [:* [:maybe ::condition-value]]](apply t2/select-one entity :id id conditions))(defn- shift-positions-after!diff --git a/src/metabase/api/macros.clj b/src/metabase/api/macros.cljindex ea041a2a8ca..3b3b81019ae 100644--- a/src/metabase/api/macros.clj+++ b/src/metabase/api/macros.clj@@ -34,6 +34,7 @@[metabase.api.open-api :as open-api][metabase.config.core :as config][metabase.events.core :as events]+ [metabase.request.schema :as request.schema][metabase.util :as u][metabase.util.log :as log][metabase.util.malli :as mu]@@ -47,15 +48,27 @@;;;; Malli schema;;;;+(mr/def ::schema-form+ "A Malli schema as written at a macro call site, unevaluated: a literal schema vector, a bare keyword (a registry+ ref or a simple type like `:string`), a symbol naming one, an expression that evaluates to one, or `nil`."+ [:or :keyword symbol? seq? vector? :nil])++(mr/def ::schema-form-or-instance+ "A schema at the point it's actually used to decode/validate/encode a value: either still a [[::schema-form]]+ (unevaluated), or already resolved to a compiled Malli schema instance."+ [:or ::schema-form [:fn {:error/message "a Malli schema"} mc/schema?]])++(mr/def ::route-regex-value+ "One `::route` regex: a compiled `Pattern`, a symbol naming one, or a form that evaluates to one."+ [:or+ (ms/InstanceOfClass java.util.regex.Pattern)+ symbol?+ seq?])+(mr/def ::route- [:map+ [:map {:closed true}[:path string?]- [:regexes {:optional true} [:map-of :keyword [:or- (ms/InstanceOfClass java.util.regex.Pattern)- ;; presumably a symbol naming a regex- symbol?- ;; presumably a form that evaluates to a regex- seq?]]]])+ [:regexes {:optional true} [:map-of :string ::route-regex-value]]])(mr/def ::param-type[:enum :route :query :body :request :respond :raise])@@ -63,22 +76,93 @@(mr/def ::params[:map-of::param-type- [:map- [:binding some?]- [:schema {:optional true} [:any {:description "Malli map schema for all the params of this type"}]]]])+ [:map {:closed true}+ [:binding ::binding-form]+ [:schema {:optional true} ::schema-form-or-instance]]])(mr/def ::method[:enum :get :post :put :delete :patch])+(mr/def ::nmspace+ "Anything `clojure.core/the-ns` accepts: a namespace object, or a symbol naming one."+ [:or symbol? (ms/InstanceOfClass clojure.lang.Namespace)])++(mr/def ::defendpoint-arg+ "One top-level form in a `defendpoint` call, before [[parse-args]] parses it: a scalar literal, a symbol, or a+ collection literal (a body form can be any Clojure expression, including a literal map/vector/set/list)."+ [:or+ :keyword+ :string+ :boolean+ :nil+ number?+ symbol?+ coll?])+(mr/def ::parsed-args- [:map+ [:map {:closed true}[:method ::method][:route ::route][:params ::params]- [:body [:sequential any?]]- [:response-schema {:optional true} [:maybe {:description "Malli schema for response. Note this is before we 'wrap if needed'"} any?]]+ [:body [:sequential ::defendpoint-arg]]+ [:response-schema {:optional true} [:maybe {:description "Malli schema for response. Note this is before we 'wrap if needed'"} ::schema-form-or-instance]][:docstr {:optional true} [:maybe string?]]- [:metadata {:optional true} [:maybe {:description "Metadata map like you'd use with `defn`"} map?]]])+ [:metadata {:optional true} [:maybe {:description "Metadata map like you'd use with `defn`"} ::route-metadata]]])++(mr/def ::conformed-schema-specifier+ [:maybe [:map {:closed true}+ [:horn [:= :-]]+ [:schema ::schema-form-or-instance]]])++(mr/def ::binding-form+ "A defendpoint parameter binding: a plain symbol, or a map/vector destructuring form."+ [:or+ symbol?+ [:sequential [:ref ::binding-form]]+ [:map-of+ [:or [:enum :as :or :keys :strs :syms] [:ref ::binding-form]]+ [:or :keyword :string symbol? [:sequential symbol?]+ [:map-of symbol? ::defendpoint-arg]]]])++(mr/def ::conformed-param+ [:map {:closed true}+ [:binding ::binding-form]+ [:schema {:optional true} ::conformed-schema-specifier]])++(mr/def ::conformed-params+ "Shape of `(:params (s/conform ::defendpoint args))`, before [[parse-params]] flattens it into [[::params]]."+ [:map {:closed true}+ [:route {:optional true} ::conformed-param]+ [:query {:optional true} ::conformed-param]+ [:body {:optional true} ::conformed-param]+ [:request {:optional true} ::conformed-param]+ [:respond-raise {:optional true} [:map {:closed true}+ [:respond symbol?]+ [:raise symbol?]]]])++(mr/def ::conformed-route+ "Shape of `(:route (s/conform ::defendpoint args))`, before [[parse-route]] transforms it into [[::route]]."+ [:multi {:dispatch first}+ [:path [:tuple [:= :path] :string]]+ [:vector [:tuple [:= :vector]+ [:map {:closed true}+ [:path :string]+ [:regexes [:sequential [:map {:closed true}+ [:key :keyword]+ [:regex ::route-regex-value]]]]]]]])++(mr/def ::conformed-args+ "Shape of [[parse-args]]'s in-progress `conformed` value by the time it reaches [[parse-route]] and+ [[inferred-route-regexes]]: `(s/conform ::defendpoint args)`, with `:params` already turned into [[::params]] by+ [[parse-params]] (which runs first), but `:route` and `:response-schema` not yet transformed."+ [:map {:closed true}+ [:method ::method]+ [:route ::conformed-route]+ [:response-schema {:optional true} ::conformed-schema-specifier]+ [:docstr {:optional true} [:maybe :string]]+ [:metadata {:optional true} [:maybe ::route-metadata]]+ [:params ::params]+ [:body [:sequential ::defendpoint-arg]]]);;; TODO -- consider whether unique key really needs to include params + regexes or not. Maybe we should just disallow;;; having two routes with the same method and param that only differ by regex patterns. It makes using this stuff more@@ -88,7 +172,7 @@[:tuple#_method ::method#_route string?- #_params [:map-of #_param keyword? #_regex-str string?]])+ #_params [:map-of #_param :string #_regex-str string?]])(mr/def ::core-fn"Schema for the underlying 'core' function generated by [[defendpoint]]. Has the form@@ -106,7 +190,7 @@[:=> [:cat any? any? any?] any?][:=> [:cat any? any? any? any?] any?]])-(mr/def ::request :map)+(mr/def ::request ::request.schema/request)(mr/def ::respond-fn[:=> [:cat any?] any?])@@ -138,7 +222,7 @@(mr/def ::info"The info about an individual endpoint that gets stored in the namespace metadata."- [:map+ [:map {:closed true}[:core-fn ::core-fn][:handler ::handler][:form ::parsed-args]])@@ -169,8 +253,8 @@:regexes (s/+ ::defendpoint.route.key-regex-pair))))))(mu/defn- infer-route-param-regex :- [:maybe (ms/InstanceOfClass java.util.regex.Pattern)]- [route-param :- :keyword- route-params-schema]+ [route-param :- :keyword+ route-params-schema :- [:maybe [:fn {:error/message "a Malli schema"} mc/schema?]]](when (and route-params-schema(= (mc/type route-params-schema) :map))(some (fn [[k _options v-schema]]@@ -178,12 +262,12 @@(second (metabase.api.common.internal/->matching-regex v-schema))))(mc/children route-params-schema))))-(mu/defn- inferred-route-regexes :- [:maybe [:map-of :keyword (ms/InstanceOfClass java.util.regex.Pattern)]]+(mu/defn- inferred-route-regexes :- [:maybe [:map-of :string (ms/InstanceOfClass java.util.regex.Pattern)]]"Auto-infer regexes for the route based on the route args schema. These are either defined by the `:api/regex`property in the schema itself (see [[metabase.api.macros-test/RouteParams]] for an example of this), or if one is notspecified, in [[metabase.api.common.internal/->matching-regex]]."[route :- :string- args :- :map]+ args :- ::conformed-args](when-let [ks (not-empty (metabase.api.common.internal/route-arg-keywords route))](let [route-params-schema (some-> (get-in args [:params :route :schema]);; eval runs at macroexpansion time to resolve the schema form@@ -195,7 +279,7 @@{}(keep (fn [k](or (when-let [regex (infer-route-param-regex k route-params-schema)]- [k regex])+ [(name k) regex]);; in dev (REPL usage) warn if we didn't infer a regex for a route param so people can consider;; adding one.(when config/is-dev?@@ -208,14 +292,14 @@ks))))(mu/defn- parse-route :- ::route- [[route-type route] :- [:tuple [:enum :path :vector] :any]- args :- :map]+ [[route-type route] :- ::conformed-route+ args :- ::conformed-args](case route-type:path (merge{:path route}(when-let [regexes (not-empty (inferred-route-regexes route args))]{:regexes regexes}))- :vector (update route :regexes #(into {} (map (juxt :key :regex)) %))))+ :vector (update route :regexes #(into {} (map (juxt (comp name :key) :regex)) %))))(s/def ::defendpoint.schema-specifier(s/?@@ -246,7 +330,7 @@[:map {:closed true}])(mu/defn- parse-params :- ::params- [params]+ [params :- ::conformed-params](letfn [(parse-schema [k param](cond(:schema param) (update param :schema :schema)@@ -274,7 +358,7 @@:body (s/* any?)))(mu/defn- parse-args :- ::parsed-args- [args :- [:sequential any?]]+ [args :- [:sequential ::defendpoint-arg]](let [conformed (s/conform ::defendpoint args)](when (= conformed :clojure.spec.alpha/invalid)(throw (ex-info (format "Unable to parse defendpoint args: %s" (s/explain-str ::defendpoint args))@@ -349,8 +433,8 @@(mu/defn validate-and-encode-response :- any?"Impl for [[endpoint-core-fn]]; validate the endpoint response against `schema` "- [schema :- some?- response :- any?]+ [schema :- ::schema-form-or-instance+ response :- ::request.schema/response](when *enable-response-validation*(when-not (mr/validate schema response)(throw (ex-info "Invalid response" ; TODO -- better error message?@@ -409,11 +493,20 @@{}(:errors explanation)))+(mr/def ::decode-params-input+ "What [[decode-and-validate-params]] is handed before decoding: the whole request itself, for a `:request`+ binding, or the raw route/query/body/form/multipart values for the others, before the endpoint's own schema+ decodes and validates them."+ [:maybe [:or+ ::request.schema/request+ ms/RingRequestParams+ ms/RingRequestBody]])+(mu/defn decode-and-validate-params"Impl for [[defendpoint]]."[params-type :- ::param-type- schema :- some?- params]+ schema :- ::schema-form-or-instance+ params :- ::decode-params-input](let [params (or params {})decoded ((decoder schema) params)](when-not (mr/validate schema decoded)@@ -610,7 +703,7 @@`(endpoint-core-fn-with-optimized-schemas(endpoint-core-fn* ~parsed-args)))-(mu/defn- params :- [:maybe [:map-of keyword? any?]]+(mu/defn- params :- [:maybe ms/RingRequestParams]"Fetch `:route` or `:query` parameters from a `request`."[request :- ::requestparams-type :- [:enum :route :query]]@@ -618,10 +711,10 @@:route (:route-params request):query (some-> (:query-params request) (update-keys keyword))))-(mu/defn- request-body+(mu/defn- request-body :- [:maybe ms/RingRequestBody]"The body params of `request`: the parts of a multipart request, the form params of a form request, or the parsedJSON body. An unparsed body (an `InputStream`) is not a param map."- [request :- :map]+ [request :- ::request](or (some-> (not-empty (:multipart-params request)) (update-keys keyword))(some-> (not-empty (:form-params request)) (update-keys keyword))(when-let [body (:body request)]@@ -701,7 +794,7 @@{:style/indent [:form]}[middleware :- [:maybe [:sequential ::middleware]]core-fn :- ::core-fn- {:keys [async?]} :- [:map+ {:keys [async?]} :- [:map {:closed true}[:async? :boolean]]](let [handler (if async?(fn async-handler [request respond raise]@@ -731,9 +824,27 @@(mr/def ::ns-endpoints [:map-of ::unique-key ::info])+(mr/def ::metadata-value-or-form+ "A `defendpoint` metadata value as written in source: either a literal, or (since metadata is an unevaluated form+ until the endpoint's namespace loads) a symbol referring to one or an expression that evaluates to one."+ [:or :string :keyword symbol? seq?])+(mr/def ::route-metadata"Metadata declared on a route via defendpoint, e.g. `{:scope \"agent:query\"}`."- :map)+ [:map {:closed true}+ [:scope {:optional true} ::metadata-value-or-form]+ [:multipart {:optional true} [:or :boolean [:map {:closed true}+ [:max-file-size {:optional true} [:or :int symbol? seq?]]+ [:max-file-count {:optional true} [:or :int symbol? seq?]]]]]+ [:deprecated {:optional true} [:or :boolean :string]]+ [:tool {:optional true} [:map {:closed true}+ [:name :string]+ [:title {:optional true} :string]+ [:description {:optional true} ::metadata-value-or-form]+ [:annotations {:optional true} [:map {:closed true}+ [:read-only? {:optional true} :boolean]+ [:idempotent? {:optional true} :boolean]]]+ [:task-support {:optional true} :keyword]]]])(mr/def ::handler-map[:map-of ::method [:sequential [:tuple@@ -751,7 +862,7 @@(m/map-vals (fn [routes](mapv (fn [route][(clout/route-compile (get-in route [:form :route :path])- (get-in route [:form :route :regexes] {}))+ (update-keys (get-in route [:form :route :regexes] {}) keyword))(:handler route)(get-in route [:form :metadata])])routes)))))@@ -795,9 +906,9 @@(mu/defn update-ns-endpoints!"Update the information about and handler stored in namespace metadata for the endpoints defined by [[defendpoint]]."- [nmspace- k :- ::unique-key- info :- ::info]+ [nmspace :- ::nmspace+ k :- ::unique-key+ info :- ::info];; we don't want to modify ns metadata during compilation, this will cause it to contain stuff like;;;; #function[metabase.api.macros/fn--61462/endpoint-handler61461--61463/f--61464]@@ -886,7 +997,7 @@([](ns-handler *ns*))- ([nmspace]+ ([nmspace :- ::nmspace](let [nmspace (the-ns nmspace)resolve-handler (fn [](-> nmspace meta :api/handler))]@@ -909,7 +1020,7 @@:else(resolve-handler))))- ([nmspace & middleware :- [:sequential {:min 1} ::middleware]]+ ([nmspace :- ::nmspace & middleware :- [:sequential {:min 1} ::middleware]](let [handler (ns-handler nmspace)](open-api/handler-with-open-api-spec(apply-middleware handler middleware)@@ -937,16 +1048,16 @@"Get the REST API endpoint handlers and forms in a namespace from its metadata."([](ns-routes *ns*))- ([nmspace]+ ([nmspace :- ::nmspace](-> nmspace the-ns meta :api/endpoints))- ([nmspace+ ([nmspace :- ::nmspacemethod :- ::method](let [routes (ns-routes nmspace)](into (empty routes)(filter (fn [[k _info]](= (first k) method)))routes)))- ([nmspace+ ([nmspace :- ::nmspacemethod :- ::methodroute :- string?](let [routes (ns-routes nmspace method)]@@ -959,14 +1070,14 @@{:description (format "Tip: you can use %s to list all the defendpoint 2 routes in a namespace" `ns-routes)}::info]"Find the info for a specific route."- [nmspace+ [nmspace :- ::nmspacemethod :- ::methodroute :- string?](first (vals (ns-routes nmspace method route))))(mu/defn find-route-fn :- ::core-fn"Find the info for a specific route."- [nmspace+ [nmspace :- ::nmspacemethod :- ::methodroute :- string?](:core-fn (find-route nmspace method route)))diff --git a/src/metabase/api/macros/defendpoint/closed_schemas.clj b/src/metabase/api/macros/defendpoint/closed_schemas.cljindex 761671ad4e0..7db15ea9c34 100644--- a/src/metabase/api/macros/defendpoint/closed_schemas.clj+++ b/src/metabase/api/macros/defendpoint/closed_schemas.clj@@ -7,140 +7,28 @@other shapes are the same escape hatch wearing a different hat and are reported too: a `:map-of` keyed by`:keyword` (or anything as loose), and `:any`, which lets a value of any shape through unexamined.- What is accepted instead:-- - a map with a `::mc/default` entry, which says what the other keys are, as long as that schema passes too;- - `:any` as a conjunct of an `:and` (or inside a normalization carrier that is one), where the other conjuncts- constrain the value and the `:any` only carries a decoder;- - the entries of a map whose `:decode/api` is [[metabase.lib.schema.common/remove-internal-keys]] that are internal- keys: the API decoder removes them from every request, so what they hold never arrives from a client;- - the schemas marked `::mr/deliberately-open`: the deliberately open maps in [[metabase.util.malli.schema]] and two- `cljc` twins in Lib, `:metabase.lib.schema.common/visualization-settings` and the non-string `:native` body of a- native stage, for values whose shape belongs to a driver, the frontend or a settings registry. Nothing else should- carry that marker.+ What is accepted is described in [[metabase.util.malli.closed-schemas]]; in addition, the entries of a map whose+ `:decode/api` is [[metabase.lib.schema.common/remove-internal-keys]] that are internal keys are skipped: the API+ decoder removes them from every request, so what they hold never arrives from a client.Registry schemas are walked once per JVM: a key already visited by an earlier endpoint is not walked again, so thefirst endpoint to reach an offending registry schema is the one that fails on it."(:require- [clojure.string :as str][malli.core :as mc]- [metabase.config.core :as config][metabase.lib.schema.common :as lib.schema.common]- [metabase.util :as u]- [metabase.util.malli.registry :as mr]))+ [metabase.util.malli.closed-schemas :as mu.closed-schemas]))(set! *warn-on-reflection* true)-(def ^:dynamic *enabled*- "Whether [[check!]] does anything. On outside prod, where the walk is a startup cost with nothing left to find."- (not config/is-prod?))--(defn- ref-name- "The registry key a schema refers to, for `[:ref ::foo]` and for a bare `::foo` reference alike."- [schema]- (if (= (mc/type schema) :ref)- (let [child (first (mc/children schema))]- (when (qualified-keyword? child) child))- (when (mc/-ref-schema? schema)- (let [form (mc/form schema)]- (when (qualified-keyword? form) form)))))--(defn- deref-safe [schema]- (let [dereffed (try (mc/deref schema) (catch Throwable _ nil))]- (when-not (identical? dereffed schema)- dereffed)))--(def ^:private open-map-of-key-schemas- "`:map-of` key schemas that make the map an open bag of arbitrary keys."- #{:keyword 'keyword? :any :some 'any? 'some?})--(def ^:private any-schemas- "Schemas that let a value of any shape through: `:any`, `:some`, and the `map?` predicate, which is an open map."- #{:any 'any? :some 'some? 'map?})--(def ^:private value-schemas- "Schemas whose children are values, not schemas, and are not walked."- #{:= :not= :enum :fn :re})--(defn- deliberately-open? [schema]- (true? (::mr/deliberately-open (mc/properties schema))))--(defn- default-entry? [child]- (and (vector? child) (= (first child) ::mc/default)))-(defn- strips-internal-keys?"Whether the API decoder removes internal keys from this map before validation."[schema](contains? #{#'lib.schema.common/remove-internal-keys lib.schema.common/remove-internal-keys}(:decode/api (mc/properties schema))))-(defn- finding [schema trail kind]- {:trail trail, :kind kind, :form (mc/form schema)})--(defn- walk-schema!- "Walk `schema`, adding to `findings`. `conjunct?` is true while inside a conjunct of an `:and`, where an `:any` is a- carrier rather than a hole; it turns false again on entering a map entry or a `:map-of` value, which are holes."- [schema trail visited findings conjunct?]- (when-let [schema (try (mc/schema schema) (catch Throwable _ nil))]- (let [schema-type (mc/type schema)- registry-key (ref-name schema)]- (cond- registry-key- (when-not (contains? @visited registry-key)- (swap! visited conj registry-key)- (when-let [dereffed (deref-safe schema)]- (walk-schema! dereffed (conj trail registry-key) visited findings conjunct?)))-- (deliberately-open? schema)- nil-- :else- (do- (cond- (and (= schema-type :map)- (not (true? (:closed (mc/properties schema))))- (not (some default-entry? (mc/children schema))))- (swap! findings conj (finding schema trail :open-map))-- (and (= schema-type :map-of)- (let [key-schema (first (mc/children schema))]- (or (contains? open-map-of-key-schemas (mc/form key-schema))- (contains? open-map-of-key-schemas (mc/type key-schema)))))- (swap! findings conj (finding schema trail :keyword-keyed-map-of))-- (and (contains? any-schemas schema-type) (not conjunct?))- (swap! findings conj (finding schema trail :any)))- (cond- (mc/-ref-schema? schema)- (when-let [dereffed (deref-safe schema)]- (walk-schema! dereffed trail visited findings conjunct?))-- (contains? value-schemas schema-type)- nil-- :else- (let [conjunct? (case schema-type- :and true- (:schema :multi) conjunct?- (:sequential :vector :set :tuple :maybe :or) false- conjunct?)- skip-key? (if (and (= schema-type :map) (strips-internal-keys? schema))- lib.schema.common/internal-key?- (constantly false))]- (doseq [[i child] (map-indexed vector (mc/children schema))]- (cond- (and (vector? child) (= 3 (count child)) (mc/schema? (nth child 2)))- (when-not (and (not (default-entry? child)) (skip-key? (first child)))- (walk-schema! (nth child 2) (conj trail (first child)) visited findings- (if (= schema-type :map)- (and conjunct? (default-entry? child))- conjunct?)))-- (= schema-type :map-of)- (walk-schema! child (conj trail i) visited findings (and conjunct? (zero? i)))-- (or (mc/schema? child) (vector? child) (qualified-keyword? child))- (walk-schema! child (conj trail i) visited findings conjunct?))))))))))+(defn- stripped-internal-key? [map-schema k]+ (and (strips-internal-keys? map-schema)+ (lib.schema.common/internal-key? k)))(def ^:private visited-registry-keys"Registry keys some endpoint has already walked, so each is walked once per JVM."@@ -153,18 +41,7 @@([schema](findings schema (atom #{})))([schema visited]- (let [found (atom [])]- (walk-schema! schema [] visited found false)- @found)))--(defn- describe-finding [{:keys [trail kind form]}]- (format " %s at %s: %s"- (case kind- :open-map "open map"- :keyword-keyed-map-of "keyword-keyed map-of"- :any ":any")- (if (seq trail) (pr-str trail) "the root")- (u/pprint-to-str form)))+ (mu.closed-schemas/findings schema {:visited visited, :skip-entry? stripped-internal-key?})))(defn check!"Throw when `schema`, the `schema-type` (`:route`, `:query`, `:body` or `:request`) schema of an endpoint, can reach@@ -172,12 +49,12 @@problem, and only a request is stripped; a `:response` schema is left alone. Impl for[[metabase.api.macros/validate-schema]]."[schema-type schema]- (when (and *enabled* (not= schema-type :response))+ (when (and mu.closed-schemas/*enabled* (not= schema-type :response))(when-let [found (not-empty (findings schema visited-registry-keys))](throw (ex-info (format (str "The %s schema of this endpoint reaches schemas that do not declare their shape."" Close every map with {:closed true}, key a map-of by :string, and give :any a"" type; a value whose keys or type really are not ours to know is one of the"" deliberately open schemas in metabase.util.malli.schema:\n%s")(name schema-type)- (str/join "\n" (map describe-finding found)))+ (mu.closed-schemas/describe-findings found)){:schema-type schema-type, :schema schema, :findings found})))))diff --git a/src/metabase/api/macros/defendpoint/open_api.clj b/src/metabase/api/macros/defendpoint/open_api.cljindex 5332841147d..a5a22c62efc 100644--- a/src/metabase/api/macros/defendpoint/open_api.clj+++ b/src/metabase/api/macros/defendpoint/open_api.clj@@ -50,8 +50,8 @@(str/replace #"[{}]" "")(str/replace #"/" "-"))))-(mu/defn- merge-required :- :metabase.api.open-api/parameter.schema.object- [schema]+(mu/defn- merge-required :- :metabase.api.open-api/parameter.schema+ [schema :- :metabase.api.open-api/parameter.schema](let [optional? (set (keep (fn [[k v]] (when (or (:optional v) (contains? v :default)) k))(:properties schema)))](-> schema@@ -70,7 +70,7 @@NOTE: maybe instead of fixing it up later we should re-work Malli's json-schema transformation into a way we want itto be?"- [schema :- :map]+ [schema :- :metabase.api.open-api/parameter.schema](try;; Helper to recursively fix nested schemas and strip :optional (which is only;; meaningful at the top level for parameter detection, not inside oneOf/anyOf/allOf)@@ -133,16 +133,48 @@{:schema schema}e)))))+(defn- normalize-raw-json-schema+ "`malli.json-schema/transform`'s raw output has `:type` as a JSON Schema string and `:properties`/`:required`/+ `:definitions` keyed by the original schema's own (keyword) keys; normalize both up front so the rest of this+ namespace can treat every JSON Schema node as [[:metabase.api.open-api/parameter.schema]] throughout."+ [schema]+ (if-not (map? schema)+ schema+ (cond-> schema+ (:type schema) (update :type keyword)+ (:format schema) (update :format keyword)+ (:description schema) (update :description str)+ (:properties schema) (update :properties (fn [props]+ (into {}+ (map (fn [[k v]] [(u/qualified-name k) (normalize-raw-json-schema v)]))+ props)))+ (:definitions schema) (update :definitions (fn [defs]+ (into {}+ (map (fn [[k v]] [(u/qualified-name k) (normalize-raw-json-schema v)]))+ defs)))+ (:required schema) (update :required (partial mapv u/qualified-name))+ (:items schema) (update :items (fn [items]+ (if (sequential? items)+ (mapv normalize-raw-json-schema items)+ (normalize-raw-json-schema items))))+ (:prefixItems schema) (update :prefixItems (partial mapv normalize-raw-json-schema))+ (map? (:additionalProperties schema)) (update :additionalProperties normalize-raw-json-schema)+ (:oneOf schema) (update :oneOf (partial mapv normalize-raw-json-schema))+ (:anyOf schema) (update :anyOf (partial mapv normalize-raw-json-schema))+ (:allOf schema) (update :allOf (partial mapv normalize-raw-json-schema)))))+(defn- mjs-collect-definitions"We transform json-schema in a few different places, but we need to collect all definitions in a single one."[malli-schema]- (let [jss (mjs/transform malli-schema {::mjs/definitions-path "#/components/schemas/"})]+ (let [jss (normalize-raw-json-schema (mjs/transform malli-schema {::mjs/definitions-path "#/components/schemas/"}))](when *definitions*(swap! *definitions* merge (:definitions (fix-json-schema jss))))(dissoc jss :definitions)))(mu/defn- schema->params* :- [:sequential :metabase.api.open-api/parameter]- [schema in-fn renames]+ [schema :- :metabase.api.macros/schema-form-or-instance+ in-fn :- ifn?+ renames :- [:maybe [:map-of :string :string]]](let [{:keys [properties required]} (mjs-collect-definitions schema)required (set required)](for [[k param-schema] properties@@ -172,7 +204,7 @@If the schema has `:openapi/response-schema` in its properties (e.g., for streaming responses), that schemais used for documentation instead of the actual schema. This allows streaming endpoints to document theJSON content they return while validating that the return value is a StreamingResponse instance."- [schema]+ [schema :- :metabase.api.macros/schema-form-or-instance](let [resolved-schema (mr/resolve-schema schema);; Check for :openapi/response-schema in the schema properties - used by server/streaming-response-schemacontent-schema (or (-> resolved-schema mc/properties :openapi/response-schema)diff --git a/src/metabase/api/open_api.clj b/src/metabase/api/open_api.cljindex 0ea3fc723bf..fc8aa9e8519 100644--- a/src/metabase/api/open_api.clj+++ b/src/metabase/api/open_api.clj@@ -61,12 +61,12 @@(list `handler-with-open-api-spec handler spec-fn)))(mr/def ::spec.info.license- [:map+ [:map {:closed true}[:name :string][:url {:optional true} :string]])(mr/def ::spec.info- [:map+ [:map {:closed true}[:title [:= "Metabase API"]][:version :string][:license {:optional true} ::spec.info.license]])@@ -78,182 +78,55 @@[:enum :get :post :put :delete :patch])(mr/def ::parameter.type- [:enum :string :number :integer :boolean :null :object :array])+ (ms/enum-decode-keyword [:string :number :integer :boolean :null :object :array]))(mr/def ::parameter.in[:enum :query :header :path :cookie])-(mr/def ::parameter.schema.common- [:map- [:default {:optional true} :any]- [:description {:optional true} :string]- [:optional {:optional true} :boolean]])--(mr/def ::parameter.schema.typed.common- [:merge- ::parameter.schema.common- [:map- [:type ::parameter.type]- ;; TODO -- I don't think `:null` can have `:enum`- [:enum {:optional true} [:sequential :any]]]])--(mr/def ::parameter.schema.string- [:merge- ::parameter.schema.typed.common- [:map- [:type [:= :string]]- [:format {:optional true} [:enum :binary "binary" :byte "byte" :uuid "uuid" :date-time "date-time"]]- [:minLength {:optional true} integer?]- [:maxLength {:optional true} integer?]- [:pattern {:optional true} (ms/InstanceOfClass java.util.regex.Pattern)]]])--(mr/def ::parameter.schema.number- [:merge- ::parameter.schema.typed.common- [:map- [:type [:= :number]]- [:minimum {:optional true} number?]- [:maximum {:optional true} number?]]])--(mr/def ::parameter.schema.integer- [:merge- ::parameter.schema.typed.common- [:map- [:type [:= :integer]]- [:minimum {:optional true} integer?]- [:maximum {:optional true} integer?]]])--(mr/def ::parameter.schema.boolean- [:merge- ::parameter.schema.typed.common- [:map- [:type [:= :boolean]]]])--(mr/def ::parameter.schema.null- [:merge- ::parameter.schema.typed.common- [:map- [:type [:= :null]]]])--(mr/def ::parameter.schema.object- [:merge- ::parameter.schema.typed.common- [:map- [:type [:= :object]]- [:properties {:optional true} [:map-of :string [:ref ::parameter.schema]]]- [:additionalProperties {:optional true} [:multi- {:dispatch boolean?}- [true [:= false]]- [false [:ref ::parameter.schema]]]]- [:required {:optional true} [:sequential :string]]- [:definitions {:optional true} [:map-of :string [:ref ::parameter.schema]]]]])--(mr/def ::parameter.schema.array- [:merge- ::parameter.schema.typed.common- [:map- [:type [:= :array]]- [:items {:optional true} [:multi- {:dispatch map?}- [true [:ref ::parameter.schema]]- ;; for a tuple. I don't think this is correct, I think you're supposed to use `prefixItems` -- see- ;; https://stackoverflow.com/questions/57464633/how-to-define-a-json-array-with-concrete-item-definition-for-every-index-i-e-a- [false [:sequential [:ref ::parameter.schema]]]]]- [:uniqueItems {:optional true} :boolean]- [:additionalItems {:optional true} :boolean] ; for tuples- [:minItems {:optional true} integer?]- [:maxItems {:optional true} integer?]]])--(mr/def ::parameter.schema.typed- [:and- [:map- [:type ::parameter.type]]- [:multi- {:dispatch :type}- [:string ::parameter.schema.string]- [:number ::parameter.schema.number]- [:integer ::parameter.schema.integer]- [:boolean ::parameter.schema.boolean]- [:null ::parameter.schema.null]- [:object ::parameter.schema.object]- [:array ::parameter.schema.array]]])--(mr/def ::parameter.schema.ref- [:merge- ::parameter.schema.common- [:map- [:$ref [:re- {:description "string starting with '#/components/schemas/'"}- #"^#/components/schemas/[^/]+$"]]- [:definitions {:optional true} [:map-of :string [:ref ::parameter.schema]]]]])--(mr/def ::parameter.schema.or- "Not sure what the difference is between `:oneOf` and `:anyOf` but they seem to both mean 'or'."- [:multi- {:dispatch (fn [x]- (if (contains? x :anyOf)- :anyOf- :oneOf))}- [:anyOf- [:merge- ::parameter.schema.common- [:map- [:anyOf [:sequential [:ref ::parameter.schema]]]]]]- [:oneOf- [:merge- ::parameter.schema.common- [:map- [:oneOf [:sequential [:ref ::parameter.schema]]]]]]])--(mr/def ::parameter.schema.and- [:merge- ::parameter.schema.common- [:map- [:allOf [:sequential [:ref ::parameter.schema]]]]])--(mr/def ::parameter.schema.const- [:merge- ::parameter.schema.common- [:map- [:const :any]]])--(mr/def ::parameter.schema.untyped-enum- [:merge- ::parameter.schema.common- [:map- [:enum [:sequential :any]]]])--(mr/def ::parameter.schema.empty- "These are mostly the result of `:fn` schemas which get translated to empty maps."- ::parameter.schema.common)-(mr/def ::parameter.schema- [:and- :map- [:multi- {:dispatch (fn [x]- (cond- (not (map? x)) :invalid- (:type x) :typed- (contains? x :$ref) :ref- (contains? x :oneOf) :or- (contains? x :anyOf) :or- (contains? x :allOf) :and- (contains? x :const) :const- (:enum x) :untyped-enum- :else :empty))}- [:invalid :map]- [:typed ::parameter.schema.typed]- [:ref ::parameter.schema.ref]- [:or ::parameter.schema.or]- [:and ::parameter.schema.and]- [:const ::parameter.schema.const]- [:untyped-enum ::parameter.schema.untyped-enum]- [:empty ::parameter.schema.empty]]])+ "A JSON Schema node as [[metabase.api.macros.defendpoint.open-api]] builds it: any combination of JSON Schema keywords."+ [:map {:closed true}+ [:type {:optional true} ::parameter.type]+ [:$ref {:optional true} [:re+ {:description "string starting with '#/components/schemas/'"}+ #"^#/components/schemas/[^/]+$"]]+ [:description {:optional true} :string]+ [:title {:optional true} :string]+ [:optional {:optional true} :boolean]+ [:default {:optional true} [:or :keyword ms/JSONSchemaLiteral]]+ [:examples {:optional true} [:sequential ms/JSONSchemaLiteral]]+ [:nullable {:optional true} :boolean]+ [:deprecated {:optional true} :boolean]+ [:const {:optional true} [:or :keyword ms/JSONSchemaLiteral]]+ [:enum {:optional true} [:sequential [:or :keyword ms/JSONSchemaLiteral]]]+ [:format {:optional true} (ms/enum-decode-keyword [:binary :byte :uuid :date-time :date :time :email :uri])]+ [:pattern {:optional true} [:or :string (ms/InstanceOfClass java.util.regex.Pattern)]]+ [:minLength {:optional true} integer?]+ [:maxLength {:optional true} integer?]+ [:minimum {:optional true} number?]+ [:maximum {:optional true} number?]+ [:exclusiveMinimum {:optional true} [:or number? :boolean]]+ [:exclusiveMaximum {:optional true} [:or number? :boolean]]+ [:multipleOf {:optional true} number?]+ [:properties {:optional true} [:map-of :string [:ref ::parameter.schema]]]+ [:required {:optional true} [:sequential :string]]+ [:additionalProperties {:optional true} [:or :boolean [:ref ::parameter.schema]]]+ [:minProperties {:optional true} integer?]+ [:maxProperties {:optional true} integer?]+ [:items {:optional true} [:or :boolean [:ref ::parameter.schema] [:sequential [:ref ::parameter.schema]]]]+ [:prefixItems {:optional true} [:sequential [:ref ::parameter.schema]]]+ [:additionalItems {:optional true} :boolean]+ [:uniqueItems {:optional true} :boolean]+ [:minItems {:optional true} integer?]+ [:maxItems {:optional true} integer?]+ [:oneOf {:optional true} [:sequential [:ref ::parameter.schema]]]+ [:anyOf {:optional true} [:sequential [:ref ::parameter.schema]]]+ [:allOf {:optional true} [:sequential [:ref ::parameter.schema]]]+ [:definitions {:optional true} [:map-of :string [:ref ::parameter.schema]]]])(mr/def ::parameter"https://swagger.io/specification/#parameter-object"- [:map+ [:map {:closed true}[:name string?][:in ::parameter.in][:description {:optional true} :string]@@ -261,26 +134,26 @@[:schema ::parameter.schema]])(mr/def ::path-item.request-body- [:map+ [:map {:closed true}[:content [:map-of[:enum "application/json" "multipart/form-data"]- [:map+ [:map {:closed true}[:schema ::parameter.schema]]]]])(mr/def ::path-item.responses[:map-of;; can be exact status codes: "200" status code ranges: "5XX" and or "default":string- [:map+ [:map {:closed true}[:description :string][:content {:optional true} [:map-of[:enum "application/json" "multipart/form-data"]- [:map [:schema ::parameter.schema]]]]+ [:map {:closed true} [:schema ::parameter.schema]]]];; TODO -- headers, links, etc.]])(mr/def ::path-item- [:map+ [:map {:closed true}[:operationId :string][:summary :string][:description :string]@@ -291,14 +164,14 @@[:responses ::path-item.responses]])(mr/def ::security-scheme- [:map+ [:map {:closed true}[:type :string][:in {:optional true} :string][:name {:optional true} :string][:description {:optional true} :string]])(mr/def ::components- [:map+ [:map {:closed true}[:schemas [:map-of :string ::parameter.schema]][:securitySchemes {:optional true} [:map-of :string ::security-scheme]]])@@ -307,7 +180,7 @@(mr/def ::spec"Based on https://swagger.io/specification/."- [:map+ [:map {:closed true}[:openapi {:optional true} :string][:info {:optional true} ::spec.info][:paths [:map-of ::path [:map-of ::method ::path-item]]]diff --git a/src/metabase/api_keys/models/api_key.clj b/src/metabase/api_keys/models/api_key.cljindex 076d68e47a3..0504343ec3a 100644--- a/src/metabase/api_keys/models/api_key.clj+++ b/src/metabase/api_keys/models/api_key.clj@@ -5,6 +5,7 @@[clojure.core.memoize :as memoize][java-time.api :as t][malli.error :as me]+ [malli.util :as mut][metabase.api-keys.core :as-alias api-keys][metabase.api-keys.db :as api-keys.db][metabase.api-keys.schema :as api-keys.schema]@@ -67,10 +68,17 @@::api-keys.schema/prefix]](subs (expose k) 0 api-keys.schema/prefix-length))+(def ^:private ApiKeyPipelineRow+ "The shape `add-prefix`/`add-key` receive: a before-insert or before-update `:model/ApiKey` instance, every column+ optional at this stage, plus the two internal keys the CRUD hooks thread through."+ (mut/merge ::api-keys.schema/api-key.update+ [:map {:closed true}+ [::api-keys/unhashed-key {:optional true} ::api-keys.schema/key.unhashed-or-secret]+ [::api-keys/group-id {:optional true} pos-int?]]))+(mu/defn- add-prefix :- [:map[:key_prefix {:optional true} ::api-keys.schema/prefix]]- [{unhashed-key ::api-keys/unhashed-key, :as api-key} :- [:map- [::api-keys/unhashed-key {:optional true} ::api-keys.schema/key.unhashed-or-secret]]]+ [{unhashed-key ::api-keys/unhashed-key, :as api-key} :- ApiKeyPipelineRow](cond-> api-key(contains? api-key ::api-keys/unhashed-key) (assoc :key_prefix (some-> unhashed-key prefix))))@@ -97,8 +105,7 @@(mu/defn- add-key"Adds the `key` based on the `:metabase.api-keys/unhashed-qkey passed in."- [{unhashed-key ::api-keys/unhashed-key, :as api-key} :- [:map- [::api-keys/unhashed-key {:optional true} ::api-keys.schema/key.unhashed-or-secret]]]+ [{unhashed-key ::api-keys/unhashed-key, :as api-key} :- ApiKeyPipelineRow](-> api-key(cond-> (contains? api-key ::api-keys/unhashed-key) (assoc :key (some-> unhashed-key hash-bcrypt)))(dissoc ::api-keys/unhashed-key)))diff --git a/src/metabase/api_keys/schema.clj b/src/metabase/api_keys/schema.cljindex 0d37de6ca86..9546861c27a 100644--- a/src/metabase/api_keys/schema.clj+++ b/src/metabase/api_keys/schema.clj@@ -110,7 +110,7 @@[:masked_key {:optional true} :string]])(defn- insert-schema [map-schema]- (into [:map]+ (into [:map {:closed true}](map (fn [[k properties schema]](if properties[k@@ -124,7 +124,7 @@(insert-schema ::api-key))(defn- update-schema [map-schema]- (into [:map]+ (into [:map {:closed true}](map (fn [[k properties schema]][k(assoc properties :optional true)diff --git a/src/metabase/app_db/cluster_lock.clj b/src/metabase/app_db/cluster_lock.cljindex 0a02a2271cd..773d549ee25 100644--- a/src/metabase/app_db/cluster_lock.clj+++ b/src/metabase/app_db/cluster_lock.clj@@ -416,11 +416,11 @@rolled-back deadlock undoes only the db writes, not external calls)."[opts :- [:or:keyword- [:map+ [:map {:closed true}[:lock {:optional true} :keyword][:locks {:optional true} [:sequential[:or :keyword- [:map+ [:map {:closed true}[:lock :keyword][:mode {:optional true} [:enum :exclusive :share]]]]]][:mode {:optional true} [:enum :exclusive :share]]@@ -471,7 +471,7 @@"Impl for [[with-detached-cluster-lock]]."[{:keys [lock timeout-seconds retry-config]:or {timeout-seconds cluster-lock-timeout-seconds}}- :- [:map+ :- [:map {:closed true}[:lock :keyword][:timeout-seconds {:optional true} :int][:retry-config {:optional true} [:ref ::retry/retry-overrides]]]diff --git a/src/metabase/app_db/connection_pool_setup.clj b/src/metabase/app_db/connection_pool_setup.cljindex 586ff129755..bc0da5c246d 100644--- a/src/metabase/app_db/connection_pool_setup.clj+++ b/src/metabase/app_db/connection_pool_setup.clj@@ -203,11 +203,12 @@"Create a connection pool [[javax.sql.DataSource]] from an unpooled [[javax.sql.DataSource]] `data-source`. If`data-source` is already pooled, this will return `data-source` as-is, ignoring `props-overrides`.`props-overrides` (if provided) are merged over [[application-db-connection-pool-props]]."- (^PoolBackedDataSource [db-type data-source]+ (^PoolBackedDataSource [db-type :- :keyword+ data-source :- (ms/InstanceOfClass javax.sql.DataSource)](connection-pool-data-source db-type data-source nil))(^PoolBackedDataSource [db-type :- :keyword^javax.sql.DataSource data-source :- (ms/InstanceOfClass javax.sql.DataSource)- props-overrides :- [:maybe [:map-of :string :any]]]+ props-overrides :- [:maybe [:map-of :string [:or :string number? :boolean]]]](if (instance? PoolBackedDataSource data-source)data-source(let [ds-name (format "metabase-%s-app-db" (name db-type))diff --git a/src/metabase/app_db/env.clj b/src/metabase/app_db/env.cljindex b13acb40593..98be879a35f 100644--- a/src/metabase/app_db/env.clj+++ b/src/metabase/app_db/env.clj@@ -40,8 +40,23 @@"postgresql" :postgres(keyword subprotocol)))))+(def ^:private EnvVars+ [:map {:closed true}+ [:mb-db-type {:optional true} [:maybe :keyword]]+ [:mb-db-in-memory {:optional true} [:maybe :boolean]]+ [:mb-db-file {:optional true} [:maybe :string]]+ [:mb-db-connection-uri {:optional true} [:maybe :string]]+ [:mb-db-host {:optional true} [:maybe :string]]+ [:mb-db-port {:optional true} [:maybe :int]]+ [:mb-db-dbname {:optional true} [:maybe :string]]+ [:mb-db-user {:optional true} [:maybe :string]]+ [:mb-db-pass {:optional true} [:maybe :string]]+ [:mb-db-azure-managed-identity-client-id {:optional true} [:maybe :string]]+ [:mb-db-aws-iam {:optional true} [:maybe :boolean]]+ [:mb-db-ssl-cert {:optional true} [:maybe :string]]])+(mu/defn- env->db-type :- [:enum :postgres :mysql :h2]- [{:keys [mb-db-connection-uri mb-db-type]}]+ [{:keys [mb-db-connection-uri mb-db-type]} :- EnvVars](or (some-> mb-db-connection-uri raw-connection-string->type)mb-db-type))diff --git a/src/metabase/app_db/setup.clj b/src/metabase/app_db/setup.cljindex 230622846a4..65138ce813f 100644--- a/src/metabase/app_db/setup.clj+++ b/src/metabase/app_db/setup.clj@@ -73,7 +73,7 @@available just in case)."[data-source :- (ms/InstanceOfClass javax.sql.DataSource)direction :- :keyword- & args]+ & args :- [:* :int]];; TODO: use [[jdbc/with-db-transaction]] instead of manually commit/rollback(with-open [conn (.getConnection ^javax.sql.DataSource data-source)](.setAutoCommit conn false)@@ -279,14 +279,15 @@state afterwards (see [[mdb.encryption/record-encryption-state!]]). Turned off by the `enable-encryption` commandand by [[metabase.cmd.copy/copy!]],which handle the encryption state themselves."- ([db-type data-source]+ ([db-type :- :keyword+ data-source :- (ms/InstanceOfClass javax.sql.DataSource)](setup-db! db-type data-source {}))([db-type :- :keyworddata-source :- (ms/InstanceOfClass javax.sql.DataSource){:keys [auto-migrate? create-sample-content? manage-encryption-state?]:or {auto-migrate? true, create-sample-content? false, manage-encryption-state? true}}- :- [:map+ :- [:map {:closed true}[:auto-migrate? {:optional true} :boolean][:create-sample-content? {:optional true} :boolean][:manage-encryption-state? {:optional true} :boolean]]]diff --git a/src/metabase/audit_app/models/audit_log.clj b/src/metabase/audit_app/models/audit_log.cljindex d0b4dacdfc3..53c07b04faf 100644--- a/src/metabase/audit_app/models/audit_log.clj+++ b/src/metabase/audit_app/models/audit_log.clj@@ -5,6 +5,7 @@(:require[clojure.data :as data][clojure.set :as set]+ [clojure.walk :as walk][metabase.api.common :as api][metabase.audit-app.db :as audit-app.db][metabase.models.interface :as mi]@@ -175,14 +176,23 @@{:previous (select-keys previous-object shared-updated-keys):new (select-keys object shared-updated-keys)}))+(defn- stringify-keys+ "Recursively turns the keyword keys of maps nested inside `x` into strings, keeping their namespaces."+ [x]+ (walk/postwalk #(cond-> % (map? %) (update-keys (fn [k] (cond-> k (keyword? k) u/qualified-name)))) x))++(def ^:private AuditedInstance+ "An audited object of any model, whose keys that model owns and `model-details` (dispatching on the model) picks from."+ [:map {:closed false, ::mr/deliberately-open true, :description "an audited entity"}])+(mr/def ::event-params [:map {:closed true:doc "Used when inserting a value to the Audit Log."}- [:object {:optional true} [:maybe :map]]- [:previous-object {:optional true} [:maybe :map]]+ [:object {:optional true} [:maybe AuditedInstance]]+ [:previous-object {:optional true} [:maybe AuditedInstance]][:user-id {:optional true} [:maybe pos-int?]][:model {:optional true} [:maybe [:or :keyword :string]]][:model-id {:optional true} [:maybe pos-int?]]- [:details {:optional true} [:maybe :map]]+ [:details {:optional true} [:maybe ms/AuditLogDetails]][:details-changed? {:optional true} [:maybe :boolean]]])(mu/defn construct-event@@ -205,11 +215,12 @@:user-id (or (:user-id params) current-user-id):model-name (model-name (or (:model params) object)):model-id (or (:model-id params) (u/id object))- :details (merge {}- (:details params)- (if (not-empty previous-object)- (prepare-update-event-data object-details previous-details)- object-details))})))+ :details (stringify-keys+ (merge {}+ (:details params)+ (if (not-empty previous-object)+ (prepare-update-event-data object-details previous-details)+ object-details)))})))(mu/defn record-event!"Records an event in the Audit Log.diff --git a/src/metabase/audit_app/schema.clj b/src/metabase/audit_app/schema.cljindex 5206d34bb62..6e3e7b3f828 100644--- a/src/metabase/audit_app/schema.clj+++ b/src/metabase/audit_app/schema.clj@@ -7,19 +7,14 @@(mr/def ::audit-log.details"The `:details` column of a AuditLog, decoded."- :map)+ ms/OpaqueJSONObject)(mr/def ::audit-log"A AuditLog as selected from the app DB: every column of `:audit_log`."- [:map {:closed true}- [:id ms/PositiveInt]- [:topic [:or :keyword :string]]- [:timestamp ms/TemporalInstant]- [:end_timestamp [:maybe ms/TemporalInstant]]- [:user_id [:maybe ::lib.schema.id/user]]- [:model [:maybe [:or :keyword :string]]]- [:model_id [:maybe :int]]- [:details ::audit-log.details]])+ [:merge+ ::audit-log.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::audit-log.update"What an update (or insert) of a AuditLog accepts: every column of `:audit_log` except `id`, all optional."diff --git a/src/metabase/auth_identity/db.clj b/src/metabase/auth_identity/db.cljindex eb87755ee93..fb6294e3f0d 100644--- a/src/metabase/auth_identity/db.clj+++ b/src/metabase/auth_identity/db.clj@@ -2,6 +2,7 @@"Application database queries for the auth identity module. Every function here is a direct Toucan 2 call with noadditional logic, so no other namespace in the module runs a query itself (model definitions still use `toucan2.core`)."(:require+ [metabase.auth-identity.schema :as auth-identity.schema][metabase.lib.schema.id :as lib.schema.id][metabase.users.schema :as users.schema][metabase.util :as u]@@ -14,8 +15,8 @@[:map {:closed true}[:user_id {:optional true} [:maybe ::lib.schema.id/user]][:provider {:optional true} [:maybe [:or :keyword :string]]]- [:credentials {:optional true} [:maybe :map]]- [:metadata {:optional true} [:maybe :map]]+ [:credentials {:optional true} [:maybe ::auth-identity.schema/auth-identity.credentials]]+ [:metadata {:optional true} [:maybe ::auth-identity.schema/auth-identity.metadata]][:provider_id {:optional true} [:maybe :string]][:last_used_at {:optional true} [:maybe ms/TemporalInstant]][:expires_at {:optional true} [:maybe ms/TemporalInstant]]diff --git a/src/metabase/auth_identity/models/auth_identity.clj b/src/metabase/auth_identity/models/auth_identity.cljindex c7d2cadd8da..d65e50d995c 100644--- a/src/metabase/auth_identity/models/auth_identity.clj+++ b/src/metabase/auth_identity/models/auth_identity.clj@@ -111,7 +111,7 @@([user-id :- ms/PositiveIntpassword :- ms/NonBlankStringopts :- [:maybe- [:map+ [:map {:closed true}[:expires-at {:optional true} [:maybe (ms/InstanceOfClass java.time.temporal.Temporal)]]]]];; always write :expires_at (nil unless an expiry was requested) so setting a password clears any stale expiry a;; prior support-access grant left behind — otherwise `authenticate` would reject the new password as expireddiff --git a/src/metabase/auth_identity/provider.clj b/src/metabase/auth_identity/provider.cljindex 551d1866985..f3bc27d59f5 100644--- a/src/metabase/auth_identity/provider.clj+++ b/src/metabase/auth_identity/provider.clj@@ -56,10 +56,12 @@[java-time.api :as t][metabase.auth-identity.db :as auth-identity.db][metabase.auth-identity.hierarchy :as auth-identity.hierarchy]+ [metabase.auth-identity.schema :as auth-identity.schema][metabase.auth-identity.session :as auth-session][metabase.events.core :as events][metabase.notification.core :as notification][metabase.premium-features.core :refer [defenterprise]]+ [metabase.users.schema :as users.schema][metabase.util :as u][metabase.util.i18n :refer [deferred-tru]][metabase.util.log :as log]@@ -254,15 +256,105 @@provider):hierarchy #'auth-identity.hierarchy/hierarchy)+(def ^:private DeviceInfo+ "Device information for session tracking, as attached to a login request."+ [:map {:closed true}+ [:device_id {:optional true} [:maybe ms/NonBlankString]]+ [:device_description {:optional true} [:maybe ms/NonBlankString]]+ [:ip_address {:optional true} [:maybe ms/NonBlankString]]+ [:embedded {:optional true} :boolean]+ [:token_exchange {:optional true} :boolean]])++(def ^:private UserData+ "SSO provider-produced data used to create or update a User during login."+ [:map {:closed true}+ [:email :string]+ [:first_name {:optional true} [:maybe :string]]+ [:last_name {:optional true} [:maybe :string]]+ [:sso_source {:optional true} :keyword]+ [:is_active {:optional true} :boolean]+ [:jwt_attributes {:optional true} [:maybe [:map-of :string [:maybe :string]]]]+ [:login_attributes {:optional true} [:maybe [:map-of :string [:maybe :string]]]]+ [:provider-id {:optional true} [:maybe :string]]+ [:tenant_id {:optional true} [:maybe ms/PositiveInt]]+ [:groups {:optional true} [:maybe [:sequential :string]]]])++(def ^:private CookieAttrs+ "One `:cookies` entry of a Ring request: a cookie's value and its attributes."+ [:map {:closed true}+ [:value {:optional true} :string]+ [:path {:optional true} :string]+ [:domain {:optional true} :string]+ [:max-age {:optional true} :int]+ [:secure {:optional true} :boolean]+ [:http-only {:optional true} :boolean]])++(def ^:private login-pipeline-entries+ "Malli map entries shared by the login pipeline map threaded through [[apply-inactive-check]] and+ [[create-session!]]: the raw Ring request keys plus every provider's request/result extension keys."+ [[:accept {:optional true} [:maybe :string]]+ [:auth-identity {:optional true} [:maybe ::auth-identity.schema/auth-identity]]+ [:authenticated-user {:optional true} [:maybe (ms/InstanceOfClass clojure.lang.IDeref)]]+ [:body {:optional true} [:maybe [:or ms/RingRequestBody (ms/InstanceOfClass java.io.InputStream)]]]+ [:browser-id {:optional true} [:maybe :string]]+ [:character-encoding {:optional true} [:maybe :string]]+ [:claims {:optional true} [:maybe ms/JWTClaims]]+ [:code {:optional true} [:maybe :string]]+ [:content-length {:optional true} [:maybe :int]]+ [:content-type {:optional true} [:maybe :string]]+ [:cookies {:optional true} [:maybe [:map-of :string CookieAttrs]]]+ [:device-info {:optional true} [:maybe DeviceInfo]]+ [:email {:optional true} [:maybe :string]]+ [:error {:optional true} [:maybe :keyword]]+ [:form-params {:optional true} [:maybe [:map-of :string [:or :string [:sequential :string]]]]]+ [:headers {:optional true} [:maybe [:map-of :string :string]]]+ [:jwt-data {:optional true} [:maybe ms/JWTClaims]]+ [:message {:optional true} [:maybe [:or :string ms/LocalizedString]]]+ [:nonce {:optional true} [:maybe :string]]+ [:oidc-nonce {:optional true} [:maybe :string]]+ [:oidc-provider {:optional true} [:maybe :keyword]]+ [:oidc-provider-key {:optional true} [:maybe :string]]+ [:params {:optional true} [:maybe ms/RingRequestParams]]+ [:password {:optional true} [:maybe :string]]+ [:path-info {:optional true} [:maybe :string]]+ [:protocol {:optional true} [:maybe :string]]+ [:provider-id {:optional true} [:maybe :string]]+ [:query-params {:optional true} [:maybe [:map-of :string [:or :string [:sequential :string]]]]]+ [:query-string {:optional true} [:maybe :string]]+ [:redirect-strategy {:optional true} [:maybe :keyword]]+ [:redirect-uri {:optional true} [:maybe :string]]+ [:redirect-url {:optional true} [:maybe :string]]+ [:remote-addr {:optional true} [:maybe :string]]+ [:request-id {:optional true} [:maybe (ms/InstanceOfClass java.util.UUID)]]+ [:request-method {:optional true} [:maybe :keyword]]+ [:route-metadata {:optional true} [:maybe :metabase.api.macros/route-metadata]]+ [:route-params {:optional true} [:maybe ms/RingRequestParams]]+ [:saml-data {:optional true} [:maybe ms/SAMLAttributes]]+ [:scheme {:optional true} [:maybe :keyword]]+ [:server-name {:optional true} [:maybe :string]]+ [:server-port {:optional true} [:maybe :int]]+ [:slack-data {:optional true} [:maybe [:map-of :string :string]]]+ [:ssl-client-cert {:optional true} [:maybe (ms/InstanceOfClass java.security.cert.X509Certificate)]]+ [:state {:optional true} [:maybe :string]]+ [:success? {:optional true} [:maybe [:or :boolean [:enum :redirect]]]]+ [:tenant-attributes {:optional true} [:maybe ms/TenantAttributes]]+ [:tenant-slug {:optional true} [:maybe :string]]+ [:token {:optional true} [:maybe :string]]+ [:token-exchange? {:optional true} :boolean]+ [:uri {:optional true} [:maybe :string]]+ [:user-data {:optional true} [:maybe UserData]]+ [:user-id {:optional true} [:maybe :int]]+ [:user-provisioning-enabled? {:optional true} [:maybe :boolean]]+ [:username {:optional true} [:maybe :string]]])+(mu/defn- apply-inactive-check"Checks if the provided `request` is an attempt to log in an active user, or an inactive one.If the user does not have `:is_active true`, the response is not successful and an error message is returned. Arequest that resolved no user at all is left alone: link-only flows legitimately finish without one."- [request :- [:map- [:user {:optional true} [:maybe [:map- [:id ms/PositiveInt]- [:is_active :boolean]]]]]]+ [request :- (into [:map {:closed true}+ [:user {:optional true} [:maybe ::users.schema/user]]]+ login-pipeline-entries)](cond-> request(and (nil? (:error request))(:user request)@@ -273,14 +365,9 @@(mu/defn- create-session!"Create a new session for a user with the given provider.Updates the last_used_at timestamp on the corresponding AuthIdentity."- [request :- [:map- [:user [:map- [:id ms/PositiveInt]- [:is_active :boolean]]]- [:device-info {:optional true} [:maybe [:map- [:device_id {:optional true} [:maybe ms/NonBlankString]]- [:device_description {:optional true} [:maybe ms/NonBlankString]]- [:ip_address {:optional true} [:maybe ms/NonBlankString]]]]]]+ [request :- (into [:map {:closed true}+ [:user ::users.schema/user]]+ login-pipeline-entries)provider :- :keyword](if-not (get-in request [:user :is_active])(assoc request :success? false@@ -364,17 +451,9 @@(mu/defn update-user!"Updates a user from user-data in the request"- [{user-id :id} :- [:map [:id ms/PositiveInt]]- user-data :- [:map- [:email :string]- [:first_name {:optional true} [:maybe :string]]- [:last_name {:optional true} [:maybe :string]]- [:sso_source {:optional true} :keyword]- [:is_active {:optional true} :boolean]- [:jwt_attributes {:optional true} [:maybe [:map-of :string [:maybe :string]]]]- [:login_attributes {:optional true} [:maybe [:map-of :string [:maybe :string]]]]- [:provider-id {:optional true} [:maybe :string]]]- provider :- :keyword]+ [{user-id :id} :- ::users.schema/user+ user-data :- UserData+ provider :- :keyword](t2/with-transaction [_](let [reactivating? (and (:is_active user-data)(not (auth-identity.db/user-active? user-id)))]@@ -388,16 +467,8 @@(mu/defn- create-user!"Create a user from user-data in the request "- [user-data :- [:map- [:email :string]- [:first_name {:optional true} [:maybe :string]]- [:last_name {:optional true} [:maybe :string]]- [:sso_source {:optional true} :keyword]- [:jwt_attributes {:optional true} [:maybe [:map-of :string [:maybe :string]]]]- [:login_attributes {:optional true} [:maybe [:map-of :string [:maybe :string]]]]- [:provider-id {:optional true} [:maybe :string]]- [:tenant_id {:optional true} [:maybe ms/PositiveInt]]]- provider :- :keyword]+ [user-data :- UserData+ provider :- :keyword](let [insert-fields (sso-user-fields)];; The tenant flow upstream validated the tenant claim and stamped :tenant_id into user-data. If;; the field list would strip it here (e.g. a premium-feature check flapped mid-request, or thediff --git a/src/metabase/auth_identity/providers/emailed_secret.clj b/src/metabase/auth_identity/providers/emailed_secret.cljindex 756311364f9..ea3f7762618 100644--- a/src/metabase/auth_identity/providers/emailed_secret.clj+++ b/src/metabase/auth_identity/providers/emailed_secret.clj@@ -6,6 +6,7 @@[metabase.auth-identity.hierarchy :as auth-identity.hierarchy][metabase.auth-identity.models.auth-identity :as auth-identity][metabase.auth-identity.provider :as provider]+ [metabase.auth-identity.schema :as auth-identity.schema][metabase.channel.email.messages :as messages][metabase.events.core :as events][metabase.util :as u]@@ -26,7 +27,8 @@and `:consumed_at` (initially nil)."[token :- :string& {:keys [expires-in-ms]- :or {expires-in-ms (* 48 60 60 1000)}}]+ :or {expires-in-ms (* 48 60 60 1000)}} :- [:maybe [:map {:closed true}+ [:expires-in-ms {:optional true} [:maybe :int]]]]]{:token_hash (u.password/hash-bcrypt token):expires_at (t/plus (t/instant) (t/millis expires-in-ms)):consumed_at nil})@@ -37,8 +39,10 @@The `email` parameter is the user's email address. Optional keyword arguments `ip-address` and `user-agent` capturerequest context information. Returns a map with `:email`, `:ip_address`, and `:request_context` (containing`:user_agent` and `:timestamp`)."- [email :- ms/Email- & {:keys [ip-address user-agent]}]+ [email :- [:maybe ms/Email]+ & {:keys [ip-address user-agent]} :- [:maybe [:map {:closed true}+ [:ip-address {:optional true} [:maybe :string]]+ [:user-agent {:optional true} [:maybe :string]]]]]{:email email:ip_address ip-address:request_context {:user_agent user-agent@@ -53,10 +57,7 @@- `:consumed` if the token has already been used- `:invalid` if the token doesn't match the stored hash"[token :- :string- credentials :- [:map- [:token_hash :string]- [:expires_at inst?]- [:consumed_at [:maybe inst?]]]]+ credentials :- ::auth-identity.schema/auth-identity.credentials.token](cond(:consumed_at credentials):consumed@@ -67,12 +68,12 @@:else:invalid))-(mu/defn mark-token-consumed :- [:map [:credentials :map]]+(mu/defn mark-token-consumed :- ::auth-identity.schema/auth-identity"Marks a token as consumed by setting the `:consumed_at` timestamp in the auth-identity's credentials.Takes an `auth-identity` map and returns an updated version with the current instant set as the `:consumed_at`value in the credentials map."- [auth-identity :- [:map [:credentials :map]]]+ [auth-identity :- ::auth-identity.schema/auth-identity](assoc-in auth-identity [:credentials :consumed_at] (t/instant)))(mu/defn- parse-token-user-id :- [:maybe ms/PositiveInt]diff --git a/src/metabase/auth_identity/schema.clj b/src/metabase/auth_identity/schema.cljindex 62d60fdade7..83c82c08e70 100644--- a/src/metabase/auth_identity/schema.clj+++ b/src/metabase/auth_identity/schema.clj@@ -5,28 +5,62 @@[metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms]))+(mr/def ::auth-identity.credentials.password+ "Credentials of the `password` provider."+ [:map {:closed true}+ [:plaintext_password {:optional true} [:maybe :string]]+ [:password_hash {:optional true} [:maybe :string]]+ [:password_salt {:optional true} [:maybe :string]]])++(mr/def ::auth-identity.credentials.token+ "Credentials of a hashed-token provider (password reset, support access grant)."+ [:map {:closed true}+ [:token_hash {:optional true} [:maybe :string]]+ [:expires_at {:optional true} [:maybe ms/TemporalInstant]]+ [:consumed_at {:optional true} [:maybe ms/TemporalInstant]]+ [:grant_ends_at {:optional true} [:maybe ms/TemporalInstant]]])++(mr/def ::auth-identity.credentials.totp+ "Credentials of the `totp` (MFA) provider."+ [:map {:closed true}+ [:secret {:optional true} :string]+ [:last_used_step {:optional true} :int]+ [:used_jtis {:optional true} [:sequential [:map {:closed true}+ [:jti :string]+ [:exp :int]]]]+ [:recovery_codes {:optional true} [:sequential :string]]+ [:email_otp {:optional true} [:map {:closed true}+ [:hash :string]+ [:exp :int]]]])+(mr/def ::auth-identity.credentials"The `:credentials` column of a AuthIdentity, decoded."- :map)+ [:or ::auth-identity.credentials.password ::auth-identity.credentials.token ::auth-identity.credentials.totp])++(mr/def ::auth-identity.metadata.emailed-secret+ "Metadata of an emailed-secret token."+ [:map {:closed true}+ [:email {:optional true} [:maybe ms/Email]]+ [:ip_address {:optional true} [:maybe :string]]+ [:request_context {:optional true} [:maybe [:map {:closed true}+ [:user_agent {:optional true} [:maybe :string]]+ [:timestamp {:optional true} [:maybe [:or ms/TemporalInstant :string]]]]]]])++(mr/def ::auth-identity.metadata.slack-connect+ "Metadata of the `slack-connect` provider."+ [:map {:closed true}+ [:signing_secret_version {:optional true} [:maybe :int]]])(mr/def ::auth-identity.metadata"The `:metadata` column of a AuthIdentity, decoded."- :map)+ [:or ::auth-identity.metadata.emailed-secret ::auth-identity.metadata.slack-connect])(mr/def ::auth-identity"A AuthIdentity as selected from the app DB: every column of `:auth_identity`."- [:map {:closed true}- [:id ms/PositiveInt]- [:user_id ::lib.schema.id/user]- [:provider [:or :keyword :string]]- [:credentials [:maybe ::auth-identity.credentials]]- [:metadata [:maybe ::auth-identity.metadata]]- [:provider_id [:maybe :string]]- [:last_used_at [:maybe ms/TemporalInstant]]- [:expires_at [:maybe ms/TemporalInstant]]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]- [:confirmed_at [:maybe ms/TemporalInstant]]])+ [:merge+ ::auth-identity.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::auth-identity.update"What an update (or insert) of a AuthIdentity accepts: every column of `:auth_identity` except `id`, all optional."diff --git a/src/metabase/auth_identity/session.clj b/src/metabase/auth_identity/session.cljindex cfa0e98ebf3..ee24f349441 100644--- a/src/metabase/auth_identity/session.clj+++ b/src/metabase/auth_identity/session.clj@@ -5,9 +5,11 @@[metabase.auth-identity.db :as auth-identity.db][metabase.login-history.core :as login-history][metabase.request.core :as request]+ [metabase.users.schema :as users.schema][metabase.util :as u][metabase.util.log :as log][metabase.util.malli :as mu]+ [metabase.util.malli.schema :as ms][metabase.util.string :as string][toucan2.core :as t2]))@@ -19,7 +21,10 @@The session row and its `login_history` row are written in one transaction: `login_history.session_id` has a foreignkey to `core_session`, so a concurrent session delete (e.g. a password change, which invalidates the user's sessions)must not be able to remove the freshly-created session between the two inserts and break that reference."- ([user device-info provider mfa-auth-identity-id]+ ([user :- ::users.schema/user+ device-info :- [:maybe request/DeviceInfo]+ provider :- :keyword+ mfa-auth-identity-id :- [:maybe ms/PositiveInt]](let [user-id (u/the-id user)provider-str (name provider)auth-identity (auth-identity.db/auth-identity-expiry user-id provider-str)@@ -37,5 +42,5 @@(assoc session:key session-key:type (if (some-> (request/current-request) request/embedded?) :full-app-embed :normal))))- ([user device-info provider]+ ([user :- ::users.schema/user device-info :- [:maybe request/DeviceInfo] provider :- :keyword](create-session-with-auth-tracking! user device-info provider nil)))diff --git a/src/metabase/batch_processing/impl.clj b/src/metabase/batch_processing/impl.cljindex cff5486fa86..ce3093d2788 100644--- a/src/metabase/batch_processing/impl.clj+++ b/src/metabase/batch_processing/impl.clj@@ -16,6 +16,7 @@[grouper.core :as grouper][metabase.app-db.core :as mdb][metabase.batch-processing.settings :as batch-processing.settings]+ [metabase.parameters.schema][metabase.util.malli :as mu][metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms])@@ -30,7 +31,7 @@[:or (ms/InstanceOfClass clojure.lang.Var) fn?])(mr/def ::grouper-wrapper- [:map+ [:map {:closed true}[:f ::fn-or-var][:grouper (ms/InstanceOfClass Grouper)]])@@ -39,9 +40,90 @@Stripped out before `f` sees it."::flush)+(def ^:private query-like+ [:fn (fn [q] (and (map? q) (or (contains? q :type) (contains? q :lib/type))))])++(def ^:private sdk-tracked-row+ [:map {:closed true}+ [:user_id {:optional true} [:maybe :int]]+ [:model {:optional true} [:maybe [:or :keyword :string]]]+ [:model_id {:optional true} [:maybe :int]]+ [:timestamp {:optional true} [:maybe [:or (ms/InstanceOfClass java.time.temporal.Temporal) :keyword]]]+ [:metadata {:optional true} [:maybe [:map {:closed true}]]]+ [:has_access {:optional true} [:maybe :boolean]]+ [:hash {:optional true} [:maybe [:or bytes? :string]]]+ [:started_at {:optional true} [:maybe (ms/InstanceOfClass java.time.temporal.Temporal)]]+ [:running_time {:optional true} [:maybe :int]]+ [:result_rows {:optional true} [:maybe :int]]+ [:native {:optional true} [:maybe :boolean]]+ [:context {:optional true} [:maybe [:or :keyword :string]]]+ [:error {:optional true} [:maybe :string]]+ [:executor_id {:optional true} [:maybe :int]]+ [:card_id {:optional true} [:maybe :int]]+ [:dashboard_id {:optional true} [:maybe :int]]+ [:pulse_id {:optional true} [:maybe :int]]+ [:database_id {:optional true} [:maybe :int]]+ [:cache_hit {:optional true} [:maybe :boolean]]+ [:action_id {:optional true} [:maybe :int]]+ [:is_sandboxed {:optional true} [:maybe :boolean]]+ [:cache_hash {:optional true} [:maybe [:or bytes? :string]]]+ [:embedding_client {:optional true} [:maybe :string]]+ [:embedding_sdk_version {:optional true} [:maybe :string]]+ [:parameterized {:optional true} [:maybe :boolean]]+ [:transform_id {:optional true} [:maybe :int]]+ [:lens_id {:optional true} [:maybe :string]]+ [:lens_params {:optional true} [:maybe [:map {:closed true} [:join_step {:optional true} [:maybe :int]]]]]+ [:auth_method {:optional true} [:maybe [:or :keyword :string]]]+ [:tenant_id {:optional true} [:maybe :int]]+ [:is_impersonated {:optional true} [:maybe :boolean]]+ [:is_db_routed {:optional true} [:maybe :boolean]]+ [:parameters {:optional true} [:maybe :string]]+ [:embedding_hostname {:optional true} [:maybe :string]]+ [:embedding_path {:optional true} [:maybe :string]]+ [:user_agent {:optional true} [:maybe :string]]+ [:ip_address {:optional true} [:maybe :string]]+ [:sanitized_user_agent {:optional true} [:maybe :string]]+ [:embedding_route {:optional true} [:maybe :string]]+ [:metabase_version {:optional true} [:maybe :string]]+ [:embedding_client_identifier {:optional true} [:maybe :string]]+ [:start_time_millis {:optional true} [:maybe :int]]+ [:json_query {:optional true} [:maybe query-like]]])++(def ^:private id+timestamp+ [:map {:closed true}+ [:id {:optional true} [:maybe :int]]+ [:timestamp {:optional true} [:maybe (ms/InstanceOfClass java.time.temporal.Temporal)]]])++(defn- satisfies-cache-backend?+ [x]+ (satisfies? @(requiring-resolve 'metabase.query-processor.middleware.cache-backend.interface/CacheBackend) x))++(def ^:private submit-object+ [:or+ [:enum :ok :not-found :invalid-format]+ [:map {:closed true}+ [:model [:enum :model/Card :model/Dashboard :model/Table :model/Document]]+ [:id :int]]+ sdk-tracked-row+ id+timestamp+ [:map {:closed true}+ [:user-id ms/PositiveInt]+ [:dashboard-id ms/PositiveInt]+ [:parameters [:sequential :metabase.parameters.schema/parameter-with-value]]]+ [:map {:closed true}+ [:user-id ms/PositiveInt]+ [:model [:enum :model/Card :model/Table :model/Dashboard :model/Collection :model/Document]]+ [:model-id ms/PositiveInt]+ [:context [:enum :view :selection]]+ [:timestamp (ms/InstanceOfClass java.time.temporal.Temporal)]]+ [:fn satisfies-cache-backend?]])+(mu/defn start! :- ::grouper-wrapper"Wrapper around [[grouper/start!]]."- [f :- ::fn-or-var & options]+ [f :- ::fn-or-var+ & options :- [:* [:or [:enum :capacity :interval :pool]+ :int+ (ms/InstanceOfClass java.util.concurrent.ExecutorService)]]];; this wrapper is so we can use Vars which Grouper normally doesn't allow.(let [f* (fn [items](when-let [items (not-empty (remove #{flush-sentinel} items))]@@ -70,7 +152,9 @@(mu/defn submit!"A wrapper of [[grouper.core/submit!]] that returns nil instead of a promise.We use grouper for fire-and-forget scenarios, so we don't care about the result."- [grouper-wrapper :- ::grouper-wrapper object & options]+ [grouper-wrapper :- ::grouper-wrapper+ object :- submit-object+ & options :- [:* [:or [:enum :callback :errback] ifn?]]](let [synchronous? (or (batch-processing.settings/synchronous-batch-updates);; if we're in the middle of a transaction, we need to do this synchronously in case we roll;; back the transaction at the end (as we do in tests)diff --git a/src/metabase/bookmarks/db.clj b/src/metabase/bookmarks/db.cljindex 643d1df57e3..1505b62c89b 100644--- a/src/metabase/bookmarks/db.clj+++ b/src/metabase/bookmarks/db.clj@@ -9,6 +9,7 @@[metabase.lib.schema.id :as lib.schema.id][metabase.util.honey-sql-2 :as h2x][metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms][toucan2.core :as t2]))@@ -210,7 +211,32 @@;; the listing. The `exploration_id` column stays so the joins in [[bookmark-rows-for-user]] resolve.{:union-all base-queries}))-(mu/defn bookmark-rows-for-user+(mr/def ::bookmark-row+ "A bookmark row left joined against the Card, Dashboard, Collection, Document, and Exploration tables."+ [:map {:closed true}+ [:created_at (ms/InstanceOfClass java.time.temporal.Temporal)]+ [:type [:enum "card" "collection" "dashboard" "document" "exploration"]]+ [:item_id ms/PositiveInt]+ [:report_card.name [:maybe :string]]+ [:report_card.card_type [:maybe :string]]+ [:report_card.display [:maybe :string]]+ [:report_card.description [:maybe :string]]+ [:report_card.archived [:maybe :boolean]]+ [:report_dashboard.name [:maybe :string]]+ [:report_dashboard.description [:maybe :string]]+ [:report_dashboard.archived [:maybe :boolean]]+ [:collection.name [:maybe :string]]+ [:collection.authority_level [:maybe :string]]+ [:collection.is_remote_synced [:maybe :boolean]]+ [:collection.description [:maybe :string]]+ [:collection.archived [:maybe :boolean]]+ [:document.name [:maybe :string]]+ [:document.archived [:maybe :boolean]]+ [:exploration.name [:maybe :string]]+ [:exploration.description [:maybe :string]]+ [:exploration.archived [:maybe :boolean]]])++(mu/defn bookmark-rows-for-user :- [:sequential ::bookmark-row]"The bookmarks of the User with `user-id`, joined against the Card, Dashboard, Collection, Document, and Explorationtables, excluding archived items, and filtered to items the target `user-scope` (a map of `:current-user-id` and`:is-superuser?`) can still read (re-checked at read time rather than trusted from when the bookmark was created,diff --git a/src/metabase/bookmarks/models/bookmark.clj b/src/metabase/bookmarks/models/bookmark.cljindex fff8bb643aa..d430cd1ad00 100644--- a/src/metabase/bookmarks/models/bookmark.clj+++ b/src/metabase/bookmarks/models/bookmark.clj@@ -3,6 +3,7 @@[clojure.string :as str][metabase.bookmarks.db :as bookmarks.db][metabase.collections.models.collection :as collection]+ [metabase.lib.schema.id :as lib.schema.id][metabase.permissions.core :as perms][metabase.queries.schema :as queries.schema][metabase.util.malli :as mu]@@ -46,7 +47,7 @@(mu/defn- normalize-bookmark-result :- BookmarkResult"Normalizes bookmark results. Bookmarks are left joined against the card, collection, dashboard, document,and exploration tables, but only points to one of them. Normalizes it so it has just the desired fields."- [result]+ [result :- ::bookmarks.db/bookmark-row](let [result (cond-> (into {} (remove (comp nil? second) result));; If not a collection then remove collection properties;; to avoid shadowing the "real" properties.@@ -74,7 +75,7 @@item_id, name, and description from the underlying bookmarked item.Bookmarks whose target `user-id` can no longer read are filtered out."- [user-id]+ [user-id :- ::lib.schema.id/user](let [user-scope {:current-user-id user-id:is-superuser? (perms/is-superuser? user-id)}](->> (bookmarks.db/bookmark-rows-for-user user-id user-scope)diff --git a/src/metabase/bookmarks/schema.clj b/src/metabase/bookmarks/schema.cljindex 2116fc181e5..c13b02bdfa1 100644--- a/src/metabase/bookmarks/schema.clj+++ b/src/metabase/bookmarks/schema.clj@@ -7,12 +7,10 @@(mr/def ::bookmark-ordering"A BookmarkOrdering as selected from the app DB: every column of `:bookmark_ordering`."- [:map {:closed true}- [:id ms/PositiveInt]- [:user_id ::lib.schema.id/user]- [:type [:or :keyword :string]]- [:item_id ms/PositiveInt]- [:ordering :int]])+ [:merge+ ::bookmark-ordering.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::bookmark-ordering.update"What an update (or insert) of a BookmarkOrdering accepts: every column of `:bookmark_ordering` except `id`, all optional."@@ -24,11 +22,10 @@(mr/def ::card-bookmark"A CardBookmark as selected from the app DB: every column of `:card_bookmark`."- [:map {:closed true}- [:id ms/PositiveInt]- [:user_id ::lib.schema.id/user]- [:card_id ::lib.schema.id/card]- [:created_at ms/TemporalInstant]])+ [:merge+ ::card-bookmark.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::card-bookmark.update"What an update (or insert) of a CardBookmark accepts: every column of `:card_bookmark` except `id`, all optional."@@ -39,11 +36,10 @@(mr/def ::collection-bookmark"A CollectionBookmark as selected from the app DB: every column of `:collection_bookmark`."- [:map {:closed true}- [:id ms/PositiveInt]- [:user_id ::lib.schema.id/user]- [:collection_id ::lib.schema.id/collection]- [:created_at ms/TemporalInstant]])+ [:merge+ ::collection-bookmark.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::collection-bookmark.update"What an update (or insert) of a CollectionBookmark accepts: every column of `:collection_bookmark` except `id`, all optional."@@ -54,11 +50,10 @@(mr/def ::dashboard-bookmark"A DashboardBookmark as selected from the app DB: every column of `:dashboard_bookmark`."- [:map {:closed true}- [:id ms/PositiveInt]- [:user_id ::lib.schema.id/user]- [:dashboard_id ::lib.schema.id/dashboard]- [:created_at ms/TemporalInstant]])+ [:merge+ ::dashboard-bookmark.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::dashboard-bookmark.update"What an update (or insert) of a DashboardBookmark accepts: every column of `:dashboard_bookmark` except `id`, all optional."@@ -69,11 +64,10 @@(mr/def ::document-bookmark"A DocumentBookmark as selected from the app DB: every column of `:document_bookmark`."- [:map {:closed true}- [:id ms/PositiveInt]- [:user_id ::lib.schema.id/user]- [:document_id ms/PositiveInt]- [:created_at ms/TemporalInstant]])+ [:merge+ ::document-bookmark.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::document-bookmark.update"What an update (or insert) of a DocumentBookmark accepts: every column of `:document_bookmark` except `id`, all optional."@@ -84,11 +78,10 @@(mr/def ::exploration-bookmark"A ExplorationBookmark as selected from the app DB: every column of `:exploration_bookmark`."- [:map {:closed true}- [:id ms/PositiveInt]- [:user_id ::lib.schema.id/user]- [:exploration_id ms/PositiveInt]- [:created_at ms/TemporalInstant]])+ [:merge+ ::exploration-bookmark.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::exploration-bookmark.update"What an update (or insert) of a ExplorationBookmark accepts: every column of `:exploration_bookmark` except `id`, all optional."diff --git a/src/metabase/cache/models/cache_config.clj b/src/metabase/cache/models/cache_config.cljindex 4e2c419ecf9..31402a7e07d 100644--- a/src/metabase/cache/models/cache_config.clj+++ b/src/metabase/cache/models/cache_config.clj@@ -22,7 +22,7 @@(def SortParams"Schema for sort parameters."- [:map+ [:map {:closed true}[:sort_column {:default :name} (into [:enum] available-sort-columns)][:sort_direction {:default :asc} [:enum :asc :desc]]])@@ -53,7 +53,7 @@("dashboard" "question")(mi/current-user-has-full-permissions?(perms/perms-objects-set-for-parent-collection- {:collection_id (target-collection-id instance)}+ (target-collection-id instance):write))))([_model pk](mi/can-write? (cache.db/cache-config pk))))@@ -145,7 +145,9 @@(mu/defn get-list"Get a list of cache configurations for given `models` and a `collection`.Supports pagination via `limit` and `offset`, and sorting via `sort-params`."- [models collection id+ [models :- [:sequential :string]+ collection :- [:maybe ms/PositiveInt]+ id :- [:maybe ms/IntGreaterThanOrEqualToZero]limit :- [:maybe ms/PositiveInt]offset :- [:maybe ms/IntGreaterThanOrEqualToZero]sort-params :- [:maybe SortParams]]@@ -158,7 +160,9 @@(mu/defn get-list-total"Get the total count of cache configurations for given `models` and a `collection`."- [models collection id]+ [models :- [:sequential :string]+ collection :- [:maybe ms/PositiveInt]+ id :- [:maybe ms/IntGreaterThanOrEqualToZero]](:count (cache.db/cache-config-count-row models collection id)))(defn store!diff --git a/src/metabase/cache/schema.clj b/src/metabase/cache/schema.cljindex 35c1cdbcc21..ee2d5f9db23 100644--- a/src/metabase/cache/schema.clj+++ b/src/metabase/cache/schema.clj@@ -5,27 +5,26 @@[metabase.util.malli.schema :as ms]))(mr/def ::cache-config.config- "The `:config` column of a CacheConfig, decoded."- :map)+ "The `:config` column of a CacheConfig, decoded: a cache strategy body with `:type` and `:refresh_automatically`+ stripped off, so it's whichever subset of these fields the strategy in the `:strategy` column calls for."+ [:map {:closed true}+ [:name {:optional true} [:maybe :string]]+ [:multiplier {:optional true} number?]+ [:min_duration_ms {:optional true} number?]+ [:duration {:optional true} number?]+ [:unit {:optional true} [:enum "hours" "minutes" "seconds" "days"]]+ [:schedule {:optional true} :string]])(mr/def ::cache-config.state"The `:state` column of a CacheConfig, decoded."- :map)+ [:map {:closed true}])(mr/def ::cache-config"A CacheConfig as selected from the app DB: every column of `:cache_config`."- [:map {:closed true}- [:id ms/PositiveInt]- [:model [:or :keyword :string]]- [:model_id [:maybe :int]]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]- [:strategy [:or :keyword :string]]- [:config ::cache-config.config]- [:state [:maybe ::cache-config.state]]- [:invalidated_at [:maybe ms/TemporalInstant]]- [:next_run_at [:maybe ms/TemporalInstant]]- [:refresh_automatically [:maybe :boolean]]])+ [:merge+ ::cache-config.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::cache-config.update"What an update (or insert) of a CacheConfig accepts: every column of `:cache_config` except `id`, all optional."@@ -43,11 +42,9 @@(mr/def ::query-cache"A QueryCache as selected from the app DB: every column of `:query_cache`."- [:map {:closed true}- [:query_hash [:or bytes? :string]]- [:updated_at ms/TemporalInstant]- [:results [:or bytes? :string]]- [:refresh_started_at [:maybe ms/TemporalInstant]]])+ [:merge+ ::query-cache.update+ [:map {:closed true}]])(mr/def ::query-cache.update"What an update (or insert) of a QueryCache accepts: every column of `:query_cache` except `id`, all optional."diff --git a/src/metabase/channel/db.clj b/src/metabase/channel/db.cljindex 14f2e628561..456e10073db 100644--- a/src/metabase/channel/db.clj+++ b/src/metabase/channel/db.clj@@ -3,6 +3,7 @@additional logic, so no other namespace in the module runs a query itself (model definitions still use `toucan2.core`)."(:require[metabase.app-db.core :as app-db]+ [metabase.channel.schema :as channel.schema][metabase.lib.schema.id :as lib.schema.id][metabase.util.malli :as mu][metabase.util.malli.schema :as ms]@@ -10,12 +11,7 @@(def ^:private ChannelRow"A whole Channel row for insert or update."- [:map {:closed true}- [:name {:optional true} [:maybe :string]]- [:description {:optional true} [:maybe :string]]- [:type {:optional true} [:maybe [:or :keyword :string]]]- [:details {:optional true} [:maybe :map]]- [:active {:optional true} [:maybe :boolean]]])+ ::channel.schema/channel.update)(mu/defn channels"Every Channel."diff --git a/src/metabase/channel/email.clj b/src/metabase/channel/email.cljindex 4d2251ed957..2955a2d7c61 100644--- a/src/metabase/channel/email.clj+++ b/src/metabase/channel/email.clj@@ -121,13 +121,23 @@{:reply-to (channel.settings/email-reply-to):from-name (channel.settings/email-from-name)}))+(def ^:private EmailAttachment+ "A postal attachment: an inline HTML body part, an inline image/logo attachment, or a CSV/XLS result attachment."+ [:map {:closed true}+ [:type {:optional true} [:maybe [:or :string :keyword]]]+ [:content {:optional true} [:maybe [:or :string (ms/InstanceOfClass java.net.URL) (ms/InstanceOfClass java.io.File)]]]+ [:content-id {:optional true} [:maybe :string]]+ [:content-type {:optional true} [:maybe :string]]+ [:file-name {:optional true} [:maybe :string]]+ [:description {:optional true} [:maybe :string]]])+(def ^:private EmailMessage[:and[:map {:closed true}[:subject :string][:recipients [:or [:sequential ms/Email] [:set ms/Email]]][:message-type [:enum :text :html :attachments]]- [:message [:or :string [:sequential :map]]]+ [:message [:or :string [:sequential EmailAttachment]]][:bcc? {:optional true} [:maybe :boolean]]][:fn {:error/message (str "Bad message-type/message combo: message-type `:attachments` should have a sequence of maps as its message; ""other types should have a String message.")}diff --git a/src/metabase/channel/email/messages.clj b/src/metabase/channel/email/messages.cljindex 6fe6243d4c3..b2cc96051e1 100644--- a/src/metabase/channel/email/messages.clj+++ b/src/metabase/channel/email/messages.clj@@ -27,7 +27,8 @@[metabase.util.i18n :as i18n :refer [trs tru]][metabase.util.json :as json][metabase.util.log :as log]- [metabase.util.malli :as mu]))+ [metabase.util.malli :as mu]+ [metabase.util.malli.schema :as ms]))(set! *warn-on-reflection* true)@@ -217,7 +218,15 @@(mu/defn send-login-from-new-device-email!"Format and send an email informing the user that this is the first time we've seen a login from this device. Expectslogin history information as returned by [[metabase.login-history.models.login-history/human-friendly-infos]]."- [{user-id :user_id, :keys [timestamp], :as login-history} :- [:map [:user_id pos-int?]]]+ [{user-id :user_id, :keys [timestamp], :as login-history} :- [:map {:closed true}+ [:user_id pos-int?]+ [:device_description {:optional true} [:maybe :string]]+ [:device_id {:optional true} [:maybe :string]]+ [:ip_address {:optional true} [:maybe :string]]+ [:location {:optional true} [:maybe :string]]+ [:session_id {:optional true} [:maybe :string]]+ [:timestamp {:optional true} [:maybe ms/TemporalInstant]]+ [:timezone {:optional true} [:maybe :string]]]](let [user-info (or (channel.db/user-contact-info user-id)(throw (ex-info (tru "User {0} does not exist" user-id){:user-id user-id, :status-code 404})))diff --git a/src/metabase/channel/email/result_attachment.clj b/src/metabase/channel/email/result_attachment.cljindex 24f2f87856e..6cbeba0c457 100644--- a/src/metabase/channel/email/result_attachment.clj+++ b/src/metabase/channel/email/result_attachment.clj@@ -1,6 +1,7 @@(ns metabase.channel.email.result-attachment(:require[clojure.java.io :as io]+ [metabase.channel.render.body :as body][metabase.channel.render.util :as render.util][metabase.channel.shared :as channel.shared][metabase.driver :as driver]@@ -35,13 +36,15 @@[^OutputStream os :- (ms/InstanceOfClass OutputStream){:keys [export-format format-rows? pivot? csv-include-bom?]:or {csv-include-bom? true}- :as _options} :- [:map+ :as _options} :- [:map {:closed true}[:export-format :keyword][:format-rows? {:optional true} [:maybe :boolean]][:pivot? {:optional true} [:maybe :boolean]][:csv-include-bom? {:optional true} [:maybe :boolean]]]- {{:keys [rows]} :data, database-id :database_id, :as results} :- [:map- [:database_id ::lib.schema.id/database]]]+ {{:keys [rows]} :data, database-id :database_id, :as results} :- [:merge+ ::body/QPResult+ [:map {:closed true}+ [:database_id ::lib.schema.id/database]]]];; make sure Database/driver info is available for the streaming results writers -- they might need this in order to;; get timezone information when writing results(log/debugf "Streaming results to %s with %d rows" export-format (:row_count results))@@ -54,7 +57,7 @@viz-settings' (assoc viz-settings :output-order output-order)](qp.si/begin! w(-> results- (assoc-in [:data :format-rows?] format-rows?)+ (assoc-in [:data :format-rows?] (boolean format-rows?))(assoc-in [:data :pivot?] pivot?)(assoc-in [:data :csv-include-bom?] csv-include-bom?)(assoc-in [:data :ordered-cols] ordered-cols))diff --git a/src/metabase/channel/render/body.clj b/src/metabase/channel/render/body.cljindex d8930ecb06b..035dda3e17f 100644--- a/src/metabase/channel/render/body.clj+++ b/src/metabase/channel/render/body.clj@@ -14,12 +14,17 @@[metabase.channel.render.table-data :as table-data][metabase.channel.render.util :as render.util][metabase.channel.settings :as channel.settings]+ [metabase.dashboards.schema][metabase.formatter.core :as formatter][metabase.geojson.api :as geojson.api][metabase.geojson.settings :as geojson.settings]+ [metabase.lib.schema :as lib.schema]+ [metabase.lib.schema.id :as lib.schema.id][metabase.models.visualization-settings :as mb.viz][metabase.pivot.core :as pivot.core][metabase.pivot.postprocess :as pivot.postprocess]+ [metabase.query-processor.compile :as qp.compile]+ [metabase.query-processor.pivot][metabase.query-processor.streaming :as qp.streaming][metabase.query-processor.streaming.common :as streaming.common][metabase.tiles.settings :as tiles.settings]@@ -72,11 +77,15 @@;;; --------------------------------------------------- Formatting ---------------------------------------------------(mu/defn- format-scalar-value- [timezone-id :- [:maybe :string] value col visualization-settings]+ "Formats a scalar card's value for rendering, whether a single value or an array-typed column's sequence of them."+ [timezone-id :- [:maybe :string]+ value :- [:or ms/FieldValue [:sequential ms/FieldValue]]+ col :- [:maybe :metabase.legacy-mbql.schema/legacy-column-metadata]+ visualization-settings :- [:maybe ms/VisualizationSettings]](cond;; legacy usage -- do not use going forward#_{:clj-kondo/ignore [:deprecated-var]}- (types/temporal-field? col)+ (types/temporal-field? (select-keys col [:base_type :effective_type]))((formatter/make-temporal-str-formatter timezone-id col {}) value)(number? value)@@ -127,25 +136,97 @@(mu/defn- query-results->row-seq"Returns a seq of stringified formatted rows that can be rendered into HTML"- [timezone-id :- [:maybe :string] visible-cols rows viz-settings]+ [timezone-id :- [:maybe :string]+ visible-cols :- [:sequential :metabase.legacy-mbql.schema/legacy-column-metadata]+ rows :- [:sequential [:sequential ms/FieldValue]]+ viz-settings :- [:maybe ms/VisualizationSettings]](let [formatters (mapv #(formatter/create-formatter timezone-id % viz-settings) visible-cols)](for [row rows]- {:row (mapv (fn [col fmt-fn]- (fmt-fn (nth row (:source-idx col) nil)))- visible-cols+ {:row (mapv (fn [value fmt-fn]+ (fmt-fn value))+ rowformatters)})))+(def ^:private TrendlineFormula+ "One of the fixed curve shapes [[metabase.analyze.fingerprint.insights/trendline-function-families]] can fit."+ [:or+ [:tuple [:= :+] number? [:tuple [:= :*] number? [:= :x]]]+ [:tuple [:= :*] number? [:tuple [:= :exp] [:tuple [:= :*] number? [:= :x]]]]+ [:tuple [:= :+] number? [:tuple [:= :*] number? [:tuple [:= :log] [:= :x]]]]+ [:tuple [:= :*] number? [:tuple [:= :pow] [:= :x] number?]]])++(def ^:private Insight+ "One entry of the `:insights` computed by `metabase.analyze.fingerprint.insights/insights`."+ [:map {:closed true}+ [:last-value {:optional true} [:maybe number?]]+ [:previous-value {:optional true} [:maybe number?]]+ [:last-change {:optional true} [:maybe number?]]+ [:slope {:optional true} [:maybe number?]]+ [:offset {:optional true} [:maybe number?]]+ [:best-fit {:optional true} [:maybe TrendlineFormula]]+ [:col {:optional true} [:maybe :string]]+ [:unit {:optional true} [:maybe :keyword]]])++(mr/def ::QPResultData+ "The `:data` of a QP result, as the render pipeline reads it."+ [:map {:closed true}+ [:cols {:optional true} [:maybe [:sequential :metabase.legacy-mbql.schema/legacy-column-metadata]]]+ [:rows {:optional true} [:maybe [:sequential [:sequential [:or ms/FieldValue [:sequential ms/FieldValue]]]]]]+ [:viz-settings {:optional true} [:maybe ms/VisualizationSettings]]+ [:results_metadata {:optional true} [:maybe [:map {:closed true}+ [:columns [:sequential :metabase.legacy-mbql.schema/legacy-column-metadata]]]]]+ [:results_timezone {:optional true} [:maybe :string]]+ [:format-rows? {:optional true} [:maybe :boolean]]+ [:native_form {:optional true} [:maybe ::qp.compile/compiled]]+ [:insights {:optional true} [:maybe [:sequential Insight]]]+ [:rows_truncated {:optional true} [:maybe :int]]+ [:csv-include-bom? {:optional true} [:maybe :boolean]]+ [:rows-file-size {:optional true} [:maybe :int]]+ [:model {:optional true} [:maybe :boolean]]+ [:dataset {:optional true} [:maybe :boolean]]+ [:pivot-export-options {:optional true} [:maybe [:map {:closed true}+ [:pivot-rows {:optional true} [:maybe [:sequential :int]]]+ [:pivot-cols {:optional true} [:maybe [:sequential :int]]]+ [:pivot-measures {:optional true} [:maybe [:sequential :int]]]+ [:show-row-totals {:optional true} :boolean]+ [:show-column-totals {:optional true} :boolean]+ [:column-sort-order {:optional true} [:maybe :metabase.query-processor.pivot/column-sort-order]]]]]])++(mr/def ::QPResult+ "A QP result map (`{:data ..., :error ...}`), as the render pipeline receives it."+ [:map {:closed true}+ [:data {:optional true} [:maybe ::QPResultData]]+ [:error {:optional true} [:maybe :string]]+ [:row_count {:optional true} [:maybe :int]]+ [:data.rows-file-size {:optional true} [:maybe :int]]+ [:notification/truncated? {:optional true} [:maybe :boolean]]+ [:status {:optional true} [:maybe [:enum :completed :failed]]]+ [:database_id {:optional true} [:maybe ::lib.schema.id/database]]+ [:started_at {:optional true} [:maybe [:or :string (ms/InstanceOfClass java.time.temporal.Temporal)]]]+ [:running_time {:optional true} [:maybe :int]]+ [:json_query {:optional true} [:maybe [:or ::lib.schema/query :metabase.legacy-mbql.schema/Query]]]+ [:average_execution_time {:optional true} [:maybe :int]]+ [:context {:optional true} [:maybe :keyword]]+ [:card_id {:optional true} [:maybe ::lib.schema.id/card]]+ [:card-error {:optional true} [:maybe :boolean]]+ [:cached {:optional true} [:maybe :string]]+ [:tenant_id {:optional true} [:maybe :int]]])+(mu/defn- prep-for-html-rendering"Convert the query results (`cols` and `rows`) into a formatted seq of rows (list of strings) that can be rendered asHTML"([timezone-id :- [:maybe :string]- card- {:keys [cols rows viz-settings], :as _data}]+ card :- [:maybe ::card]+ {:keys [cols rows viz-settings], :as _data} :- ::QPResultData](let [visible-cols (table-data/visible-columns cols)row-limit (min (channel.settings/attachment-table-row-limit) 100)](cons(query-results->header-row card visible-cols)- (query-results->row-seq timezone-id visible-cols (take row-limit rows) viz-settings)))))+ (query-results->row-seq timezone-id+ (mapv #(dissoc % :source-idx :remapped_to_column) visible-cols)+ (for [row (take row-limit rows)]+ (mapv #(nth row (:source-idx %) nil) visible-cols))+ viz-settings)))))(defn- strong-limit-text [number][:strong {:style (style/style {:color style/color-gray-3})} (h (formatter/format-scalar-number number))])@@ -174,11 +255,65 @@;;; | render |;;; +----------------------------------------------------------------------------------------------------------------++(mr/def ::hiccup+ "A node of a Hiccup form: a scalar, a `[tag attrs? & children]` vector (left unstructured -- HTML attribute keys+ are open-ended), or a seq of such nodes."+ [:or :string number? :boolean nil? vector? [:sequential [:ref ::hiccup]]])++(mr/def ::adhoc-card+ "Schema for an ad-hoc (unsaved) card."+ [:map {:closed true}+ [:display :keyword]+ [:visualization_settings {:optional true} [:maybe ms/VisualizationSettings]]+ [:name {:optional true} [:maybe :string]]])++(mr/def ::card+ "A card as `render`ed for a Pulse/Dashboard Subscription: either a real Card, or an ad-hoc (unsaved) card."+ [:or+ [:merge+ :metabase.queries.schema/card+ [:map {:closed true}+ [:include_csv {:optional true} [:maybe :boolean]]+ [:include_xls {:optional true} [:maybe :boolean]]+ [:format_rows {:optional true} [:maybe :boolean]]+ [:pivot_results {:optional true} [:maybe :boolean]]]]+ ::adhoc-card])++(mr/def ::dashcard+ "A DashboardCard as `render`ed for a Dashboard Subscription, plus the `:series-results` key+ `notification.payload.execute` attaches for multi-series cards."+ [:merge+ :metabase.dashboards.schema/dashboard-card+ [:map {:closed true}+ [:series-results {:optional true} [:maybe [:sequential+ [:map {:closed true}+ [:type {:optional true} [:= :card]]+ [:card {:optional true} [:maybe [:ref :metabase.queries.schema/card]]]+ [:dashcard {:optional true} [:maybe [:ref :metabase.dashboards.schema/dashboard-card]]]+ [:result {:optional true} [:maybe ::QPResult]]]]]]]])++(mr/def ::render-type+ [:enum :inline :attachment])++;;; I gave these keys below namespaces to make them easier to find usages for but didn't use `metabase.channel.render` so+;;; we can keep this as an internal namespace you don't need to know about outside of the module.+(mr/def ::options+ "Options for Pulse (i.e. Alert/Dashboard Subscription) rendering."+ [:map {:closed true}+ [:channel.render/include-buttons? {:description "default: false", :optional true} :boolean]+ [:channel.render/include-title? {:description "default: false", :optional true} :boolean]+ [:channel.render/include-description? {:description "default: false", :optional true} :boolean]+ [:channel.render/disable-links? {:description "default: false", :optional true} :boolean]+ [:channel.render/include-inline-parameters? {:description "default: false", :optional true} :boolean]+ [:channel.render/padding-x {:description "default: 0, horizontal pixels around image", :optional true} [:maybe :int]]+ [:channel.render/padding-y {:description "default: 0, vertical pixels around image", :optional true} [:maybe :int]]+ [:channel.render/scale {:description "default: 1.0, a factor, or a fn of the laid-out content's logical width/height returning one", :optional true} [:maybe [:or number? ifn?]]]])+(mr/def ::RenderedPartCard"Schema used for functions that operate on pulse card contents and their attachments"- [:map+ [:map {:closed true}[:attachments {:optional true} [:maybe [:map-of :string (ms/InstanceOfClass URL)]]]- [:content [:sequential :any]]+ [:content ::hiccup][:render/text {:optional true} [:maybe :string]]])(defmulti render@@ -230,7 +365,8 @@minibar-cols (minibar-columns (get-in unordered-data [:results_metadata :columns] []) viz-settings)table-body [:div(table/render-table- (select-keys unordered-data [:cols :rows])+ {:cols (mapv #(select-keys % [:name]) (:cols unordered-data))+ :rows (:rows unordered-data)}{:cols-for-color-lookup (mapv :name filtered-cols):col-names (streaming.common/column-titles filtered-cols viz-settings format-rows?)}(prep-for-html-rendering timezone-id card data)@@ -743,7 +879,7 @@(let [base-rows (into [] (comp (filter #(zero? (nth % pg-idx)))(map #(vec (m/remove-nth pg-idx %))))(:rows data))]- {:cols columns :rows base-rows}))]+ {:cols (mapv #(select-keys % [:name]) columns) :rows base-rows}))];; Unlike the flat :table path, a pivot aggregates all rows into a bounded grid rather than;; truncating displayed rows, so the flat-table row-count truncation warning doesn't apply.(pivot->hiccup output {:color-data color-datadiff --git a/src/metabase/channel/render/card.clj b/src/metabase/channel/render/card.cljindex 12f50ed06ed..2aa80bb00b4 100644--- a/src/metabase/channel/render/card.clj+++ b/src/metabase/channel/render/card.clj@@ -18,33 +18,16 @@[metabase.util.malli.registry :as mr][metabase.util.markdown :as markdown]))-;;; I gave these keys below namespaces to make them easier to find usages for but didn't use `metabase.channel.render` so-;;; we can keep this as an internal namespace you don't need to know about outside of the module.-(mr/def ::options- "Options for Pulse (i.e. Alert/Dashboard Subscription) rendering."- [:map- [:channel.render/include-buttons? {:description "default: false", :optional true} :boolean]- [:channel.render/include-title? {:description "default: false", :optional true} :boolean]- [:channel.render/include-description? {:description "default: false", :optional true} :boolean]- [:channel.render/disable-links? {:description "default: false", :optional true} :boolean]- [:channel.render/include-inline-parameters? {:description "default: false", :optional true} :boolean]- [:channel.render/padding-x {:description "default: 0, horizontal pixels around image", :optional true} [:maybe :int]]- [:channel.render/padding-y {:description "default: 0, vertical pixels around image", :optional true} [:maybe :int]]])--(mr/def ::adhoc-card- "Schema for an ad-hoc (unsaved) card."- [:map- [:display :keyword]- [:visualization_settings {:optional true} [:maybe :map]]- [:name {:optional true} [:maybe :string]]])-(defn- card-href[card](when-let [card-id (u/id card)](h (urls/card-url card-id))))(mu/defn- make-title-if-needed :- [:maybe ::body/RenderedPartCard]- [render-type card dashcard options :- [:maybe ::options]]+ [render-type :- ::body/render-type+ card :- [:maybe ::body/card]+ dashcard :- [:maybe ::body/dashcard]+ options :- [:maybe ::body/options]](when (:channel.render/include-title? options)(let [card-name (render.util/dashcard-title card dashcard)image-bundle (when (:channel.render/include-buttons? options)@@ -74,7 +57,9 @@:src (:image-src image-bundle)}])]]]]})))(mu/defn- make-description-if-needed :- [:maybe ::body/RenderedPartCard]- [dashcard card options :- [:maybe ::options]]+ [dashcard :- [:maybe ::body/dashcard]+ card :- [:maybe ::body/card]+ options :- [:maybe ::body/options]](when (:channel.render/include-description? options)(when-let [description (or (get-in dashcard [:visualization_settings :card.description])(:description card))]@@ -205,11 +190,11 @@((some-fn :include_csv :include_xls) card))(mu/defn- render-pulse-card-body :- ::body/RenderedPartCard- [render-type+ [render-type :- ::body/render-typetimezone-id :- [:maybe :string]- card- dashcard- {:keys [data error] :as results}]+ card :- [:maybe ::body/card]+ dashcard :- [:maybe ::body/dashcard]+ {:keys [data error] :as results} :- [:maybe ::body/QPResult]](try(when error(throw (ex-info (tru "Card has errors: {0}" error) (assoc results :card-error true))))@@ -249,15 +234,19 @@- content (a hiccup form suitable for rendering on rich clients or rendering into an image)- render/text : raw text suitable for substituting on clients when text is preferable. (Currently slack uses this forscalar results where text is preferable to an image of a div of a single result."- ([render-type timezone-id card dashcard results]+ ([render-type :- ::body/render-type+ timezone-id :- [:maybe :string]+ card :- [:maybe ::body/card]+ dashcard :- [:maybe ::body/dashcard]+ results :- [:maybe ::body/QPResult]](render-pulse-card render-type timezone-id card dashcard results nil))- ([render-type+ ([render-type :- ::body/render-typetimezone-id :- [:maybe :string]- card- dashcard- results- options :- [:maybe ::options]]+ card :- [:maybe ::body/card]+ dashcard :- [:maybe ::body/dashcard]+ results :- [:maybe ::body/QPResult]+ options :- [:maybe ::body/options]](let [{title :contenttitle-attachments :attachments} (make-title-if-needed render-type card dashcard options){description :content} (make-description-if-needed dashcard card options)@@ -299,20 +288,34 @@(mu/defn render-pulse-card-for-display"Same as `render-pulse-card` but isn't intended for an email, rather for previewing so there is no need forattachments"- ([timezone-id card results]+ ([timezone-id :- [:maybe :string]+ card :- [:maybe ::body/card]+ results :- [:maybe ::body/QPResult]](render-pulse-card-for-display timezone-id card results nil))- ([timezone-id card results options :- [:maybe ::options]]+ ([timezone-id :- [:maybe :string]+ card :- [:maybe ::body/card]+ results :- [:maybe ::body/QPResult]+ options :- [:maybe ::body/options]](:content (render-pulse-card :inline timezone-id card nil results options))))+(mr/def ::part+ "The `{:card :dashcard :result}` shape a Pulse/Dashboard Subscription section carries."+ [:map {:closed true}+ [:type {:optional true} [:= :card]]+ [:card {:optional true} [:maybe ::body/card]]+ [:dashcard {:optional true} [:maybe ::body/dashcard]]+ [:result {:optional true} [:maybe ::body/QPResult]]])+(mu/defn render-pulse-section :- ::body/RenderedPartCard"Render a single Card section of a Pulse to a Hiccup form (representing HTML)."- ([timezone-id part]+ ([timezone-id :- [:maybe :string]+ part :- ::part](render-pulse-section timezone-id part {}))- ([timezone-id- {card :card, dashcard :dashcard, result :result, :as _part}- options :- [:maybe ::options]]+ ([timezone-id :- [:maybe :string]+ {card :card, dashcard :dashcard, result :result, :as _part} :- ::part+ options :- [:maybe ::body/options]](log/with-context {:card_id (:id card)}(let [options (merge {:channel.render/include-title? true:channel.render/include-description? true@@ -326,25 +329,30 @@(mu/defn render-pulse-card-to-png :- bytes?"Render a `pulse-card` as a PNG. `data` is the `:data` from a QP result."- (^bytes [timezone-id pulse-card result width]+ (^bytes [timezone-id :- [:maybe :string]+ pulse-card :- [:maybe ::body/card]+ result :- ::body/QPResult+ width :- pos-int?](render-pulse-card-to-png timezone-id pulse-card result width nil))(^bytes [timezone-id :- [:maybe :string]- pulse-card- result- width- options :- [:maybe ::options]]+ pulse-card :- [:maybe ::body/card]+ result :- ::body/QPResult+ width :- pos-int?+ options :- [:maybe ::body/options]](png/render-html-to-png (render-pulse-card :inline timezone-id pulse-card nil result options) width options)))(mu/defn render-adhoc-card-to-png :- bytes?"Render an ad-hoc (unsaved) card to PNG."- (^bytes [adhoc-card results width]+ (^bytes [adhoc-card :- ::body/adhoc-card+ results :- ::body/QPResult+ width :- pos-int?](render-adhoc-card-to-png adhoc-card results width nil))- (^bytes [adhoc-card :- ::adhoc-card- results :- [:map [:data :map]]- width- options :- [:maybe ::options]]+ (^bytes [adhoc-card :- ::body/adhoc-card+ results :- ::body/QPResult+ width :- pos-int?+ options :- [:maybe ::body/options]](let [timezone-id (qp.timezone/system-timezone-id)](png/render-html-to-png(render-pulse-card :inline timezone-id adhoc-card nil results options)@@ -353,7 +361,11 @@(mu/defn render-pulse-card-to-base64 :- string?"Render a `pulse-card` as a PNG and return it as a base64 encoded string."- ^String [timezone-id card dashcard result width]+ ^String [timezone-id :- [:maybe :string]+ card :- [:maybe ::body/card]+ dashcard :- [:maybe ::body/dashcard]+ result :- ::body/QPResult+ width :- pos-int?](-> (render-pulse-card :inline timezone-id card dashcard result)(png/render-html-to-png width)image-bundle/render-img-data-uri))@@ -361,14 +373,17 @@(mu/defn png-from-render-info :- bytes?"Create a PNG file (as a byte array) from rendering info. `options` takes the keys[[metabase.channel.render.png/render-html-to-png]] reads, e.g. `:channel.render/scale`."- (^bytes [rendered-info :- ::body/RenderedPartCard width]+ (^bytes [rendered-info :- ::body/RenderedPartCard+ width :- pos-int?];; TODO huh? why do we need this indirection?(png/render-html-to-png rendered-info width))- (^bytes [rendered-info :- ::body/RenderedPartCard width options]+ (^bytes [rendered-info :- ::body/RenderedPartCard+ width :- pos-int?+ options :- [:maybe ::body/options]](png/render-html-to-png rendered-info width options)))(mu/defn defaulted-timezone :- :string"Returns the timezone ID for the given `card`. Either the report timezone (if applicable) or the JVM timezone."- [card]+ [card :- [:maybe ::body/card]](or (some->> card :database_id channel.db/database qp.timezone/results-timezone-id)(qp.timezone/system-timezone-id)))diff --git a/src/metabase/channel/render/js/color.clj b/src/metabase/channel/render/js/color.cljindex 762097d2f3e..7523e6ed2bc 100644--- a/src/metabase/channel/render/js/color.clj+++ b/src/metabase/channel/render/js/color.clj@@ -5,7 +5,8 @@[metabase.channel.render.js.protocol :as js.protocol][metabase.channel.render.js.renderer :as renderer][metabase.formatter.core :as formatter]- [metabase.util.malli :as mu]))+ [metabase.util.malli :as mu]+ [metabase.util.malli.schema :as ms]))(set! *warn-on-reflection* true)@@ -13,15 +14,21 @@"This is a pretty loose schema, more as a safety net as we have a long feedback loop for this being broken as it'sbeing handed to the JS color picking code. Currently it just needs column names from `:cols`, and the query resultsfrom `:rows`"- [:map- [:cols [:sequential [:map+ [:map {:closed true}+ [:cols [:sequential [:map {:closed true}[:name :string]]]]- [:rows [:sequential [:sequential :any]]]])+ [:rows [:sequential [:sequential+ [:or ms/FieldValue+ (ms/InstanceOfClass metabase.formatter.impl.NumericWrapper)+ (ms/InstanceOfClass metabase.formatter.impl.TextWrapper)]]]]])(def ^:private Cell"One cell to color: its value (possibly a formatter wrapper), its row index (row-highlight rules read the whole rowfrom the `:rows` passed alongside), and its column name."- [:tuple :any :int [:maybe :string]])+ [:tuple [:or ms/FieldValue+ (ms/InstanceOfClass metabase.formatter.impl.NumericWrapper)+ (ms/InstanceOfClass metabase.formatter.impl.TextWrapper)]+ :int [:maybe :string]])(defn- ->js-number"Coerce BigDecimal/BigInteger to primitive double/long so the JSON handed to the JS side carries plain numbers.@@ -57,7 +64,7 @@string (hex or `rgba()`) or nil for cells no rule matches. The coloring rules come from `:table.column_formatting`in `viz-settings` (row-highlight rules are disabled when `:table.pivot` is set)."[{:keys [cols rows]} :- QueryResults- viz-settings+ viz-settings :- [:maybe ms/VisualizationSettings]cells :- [:sequential Cell]](if (or (empty? cells)(empty? (or (:table.column_formatting viz-settings)diff --git a/src/metabase/channel/render/png.clj b/src/metabase/channel/render/png.cljindex 3c4251d69a2..567140b2c14 100644--- a/src/metabase/channel/render/png.clj+++ b/src/metabase/channel/render/png.clj@@ -170,12 +170,12 @@(mu/defn render-html-to-png :- bytes?"Render the Hiccup HTML `content` of a Pulse to a PNG image, returning a byte array."(^bytes [rendered-info :- ::body/RenderedPartCard- width]+ width :- pos-int?](render-html-to-png rendered-info width nil))(^bytes [{:keys [content]} :- ::body/RenderedPartCard- width- options]+ width :- pos-int?+ options :- [:maybe ::body/options]](try(let [padding-x (or (:channel.render/padding-x options) 0)padding-y (or (:channel.render/padding-y options) 0)diff --git a/src/metabase/channel/schema.clj b/src/metabase/channel/schema.cljindex 8b0fb8d68d3..295d98a6fec 100644--- a/src/metabase/channel/schema.clj+++ b/src/metabase/channel/schema.clj@@ -22,8 +22,10 @@[:channel :string]])(mr/def ::email-details- "The connection `:details` of a `:channel/email` channel: there are none, email is configured by settings."- [:map {:closed true}])+ "The connection `:details` of a `:channel/email` channel: an optional SMTP host and port, as email is otherwise configured by settings."+ [:map {:closed true}+ [:host {:optional true} [:maybe :string]]+ [:port {:optional true} [:maybe :int]]])(mr/def ::test-details"The `:details` of the test-only `:channel/metabase-test` channel, which exists so a test can drive a connection test@@ -143,13 +145,10 @@(mr/def ::channel-template"A ChannelTemplate as selected from the app DB: every column of `:channel_template`."- [:map {:closed true}- [:id ms/PositiveInt]- [:name :string]- [:channel_type [:or :keyword :string]]- [:details [:maybe ::channel-template.details]]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]])+ [:merge+ ::channel-template.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::channel-template.update"What an update (or insert) of a ChannelTemplate accepts: every column of `:channel_template` except `id`, all optional."diff --git a/src/metabase/channel/slack.clj b/src/metabase/channel/slack.cljindex 6a75849fe2f..99967be45f0 100644--- a/src/metabase/channel/slack.clj+++ b/src/metabase/channel/slack.clj@@ -13,6 +13,7 @@[metabase.util.json :as json][metabase.util.log :as log][metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms]))(set! *warn-on-reflection* true)@@ -348,7 +349,9 @@channel ID, a user ID, or a legacy display name (\"#general\"/\"@bob\") resolved via the cached channel/user list.Either way the file arrives as one message with `initial-comment` (mrkdwn, optional) as its caption. A channel isjoined first, since sharing into one requires membership; a user's DM is opened by Slack itself."- ([file filename channel-id]+ ([file :- NonEmptyByteArray+ filename :- ms/NonBlankString+ channel-id :- ms/NonBlankString](upload-file-to-channel! file filename channel-id nil))([file :- NonEmptyByteArrayfilename :- ms/NonBlankString@@ -362,6 +365,28 @@(join-channel! target)(complete-upload! file filename {:channel-id target} initial-comment))))))+(mr/def ::slack-block+ "A Slack Block Kit block, element, or legacy `attachments` entry -- self-similar and recursive (blocks nest+ elements, elements nest elements, and `:text`/`:accessory` nest another block-shaped object).+ See https://api.slack.com/block-kit."+ [:map {:closed true}+ [:type {:optional true} [:maybe :string]]+ [:text {:optional true} [:maybe [:or :string [:ref ::slack-block]]]]+ [:fields {:optional true} [:maybe [:sequential [:ref ::slack-block]]]]+ [:elements {:optional true} [:maybe [:sequential [:ref ::slack-block]]]]+ [:accessory {:optional true} [:maybe [:ref ::slack-block]]]+ [:block_id {:optional true} [:maybe :string]]+ [:image_url {:optional true} [:maybe :string]]+ [:alt_text {:optional true} [:maybe :string]]+ [:emoji {:optional true} [:maybe :boolean]]+ [:color {:optional true} [:maybe :string]]+ [:pretext {:optional true} [:maybe :string]]+ [:title {:optional true} [:maybe :string]]+ [:author_name {:optional true} [:maybe :string]]+ [:footer {:optional true} [:maybe :string]]+ [:fallback {:optional true} [:maybe :string]]+ [:ts {:optional true} [:maybe [:or :string number?]]]])+(mu/defn post-chat-message!"Calls Slack API `chat.postMessage` endpoint and posts a message to a channel.message-blocks if provided should be a map containing slack message blocks@@ -369,9 +394,9 @@See: https://app.slack.com/block-kit-builder"[message-content :- [:map {:closed true}[:channel :string]- [:blocks {:optional true} [:sequential :map]]+ [:blocks {:optional true} [:sequential ::slack-block]][:text {:optional true} :string]- [:attachments {:optional true} [:sequential :map]]]]+ [:attachments {:optional true} [:sequential ::slack-block]]]];; TODO: it would be nice to have an emoji or icon image to use here(let [base-params {:username "Metabot":icon_url "http://static.metabase.com/metabot_slack_avatar_whitebg.png"}diff --git a/src/metabase/cloud_migration/schema.clj b/src/metabase/cloud_migration/schema.cljindex 4779150888f..9e52845edcd 100644--- a/src/metabase/cloud_migration/schema.clj+++ b/src/metabase/cloud_migration/schema.clj@@ -6,14 +6,10 @@(mr/def ::cloud-migration"A CloudMigration as selected from the app DB: every column of `:cloud_migration`."- [:map {:closed true}- [:id ms/PositiveInt]- [:external_id :string]- [:upload_url :string]- [:state [:or :keyword :string]]- [:progress :int]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]])+ [:merge+ ::cloud-migration.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::cloud-migration.update"What an update (or insert) of a CloudMigration accepts: every column of `:cloud_migration` except `id`, all optional."diff --git a/src/metabase/collections/children.clj b/src/metabase/collections/children.cljindex 5ec3007872e..29d7ccf1a7f 100644--- a/src/metabase/collections/children.clj+++ b/src/metabase/collections/children.clj@@ -37,6 +37,7 @@[metabase.util :as u][metabase.util.honey-sql-2 :as h2x][metabase.util.malli :as mu]+ [metabase.util.malli.schema :as ms][metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay][toucan2.core :as t2]))@@ -199,7 +200,7 @@(def CollectionChildrenOptions"The `options` map accepted by [[collection-children]] and [[collection-filter-metadata]]."- [:map+ [:map {:closed true}[:show-dashboard-questions? :boolean][:show-exploration-documents? {:optional true} [:maybe :boolean]][:collection-type {:optional true} [:maybe CollectionType]]@@ -212,7 +213,7 @@;; column (see [[creator-filterable-models]]) return nothing.[:created-by-id {:optional true} [:maybe pos-int?]][:search-text {:optional true} [:maybe :string]]- [:sort-info {:optional true} [:maybe [:map+ [:sort-info {:optional true} [:maybe [:map {:closed true}[:sort-column (into [:enum {:error/message "sort-columns"}](map normalize-sort-choice)valid-sort-columns)]@@ -591,7 +592,13 @@;; internal-only: stamped in `post-process-card-row` for the permission check:document_id)(update :dashboard #(when % (select-keys % [:id :name :moderation_status])))- (assoc :fully_parameterized (queries/fully-parameterized? row))))+ (assoc :fully_parameterized (queries/fully-parameterized? (select-keys row [:dataset_query])))))++(defn- assoc-based-on-upload+ "Assoc `:based_on_upload` onto each of the model `rows` that is based on an upload."+ [rows]+ (let [id->table-id (upload/models-based-on-upload (map #(select-keys % upload/based-on-upload-input-keys) rows))]+ (map #(m/assoc-some % :based_on_upload (id->table-id (:id %))) rows)))(defn- post-process-card-like[{:keys [hydrate-based-on-upload]} rows]@@ -605,7 +612,7 @@(as-> (map post-process-card-row rows) $(apply t2/hydrate $ hydration)(cond-> $- hydrate-based-on-upload upload/model-hydrate-based-on-upload)+ hydrate-based-on-upload assoc-based-on-upload)(map post-process-card-row-after-hydrate $))))(defmethod post-process-collection-children :card@@ -874,23 +881,30 @@(t2/hydrate tables :measures)tables)))+(def ^:private last-edit-key-mapping+ {:last_edit_user :id+ :last_edit_last_name :last_name+ :last_edit_first_name :first_name+ :last_edit_email :email+ :last_edit_timestamp :timestamp})+;;; TODO -- consider whether this function belongs here or in [[metabase.revisions.models.revision.last-edit]](mu/defn- coalesce-edit-info :- revisions/MaybeAnnotated"Hoist all of the last edit information into a map under the key :last-edit-info. Considers this information presentif `:last_edit_user` is not nil."- [row]+ [row :- [:map {:closed true}+ [:last_edit_user {:optional true} [:maybe :int]]+ [:last_edit_last_name {:optional true} [:maybe :string]]+ [:last_edit_first_name {:optional true} [:maybe :string]]+ [:last_edit_email {:optional true} [:maybe :string]]+ [:last_edit_timestamp {:optional true} [:maybe ms/TemporalInstant]]]](letfn [(select-as [original k->k'](reduce (fn [m [k k']] (assoc m k' (get original k))){}k->k'))]- (let [mapping {:last_edit_user :id- :last_edit_last_name :last_name- :last_edit_first_name :first_name- :last_edit_email :email- :last_edit_timestamp :timestamp}]- (cond-> (apply dissoc row (keys mapping))- ;; don't use contains as they all have the key, we care about a value present- (:last_edit_user row) (assoc :last-edit-info (select-as row mapping))))))+ (cond-> {}+ ;; don't use contains as they all have the key, we care about a value present+ (:last_edit_user row) (assoc :last-edit-info (select-as row last-edit-key-mapping)))))(defn- remove-unwanted-keys [{:keys [model] :as row}](cond-> (dissoc row :model_ranking :archived_directly :total_count :collection_type)@@ -921,7 +935,9 @@(comp (map (fn [[model rows]](post-process-collection-children (keyword model) options collection rows)))cat- (map coalesce-edit-info)))+ (map (fn [row]+ (merge (apply dissoc row (keys last-edit-key-mapping))+ (coalesce-edit-info (select-keys row (keys last-edit-key-mapping))))))))(map remove-unwanted-keys);; the collection these are presented "in" is the ID of the collection we're getting `/items` on.(map #(assoc % :collection_id (:id collection)))diff --git a/src/metabase/collections/create.clj b/src/metabase/collections/create.cljindex 2d6106c151b..f41c4ac3983 100644--- a/src/metabase/collections/create.clj+++ b/src/metabase/collections/create.clj@@ -42,7 +42,7 @@(def ^:private CreateCollectionArguments"The arguments to a create-collection call — what the public API surface accepts."- [:map+ [:map {:closed true}[:name ms/NonBlankString][:description {:optional true} [:maybe ms/NonBlankString]][:parent_id {:optional true} [:maybe ms/PositiveInt]]@@ -88,7 +88,7 @@superuser/feature check, runs the EE tenant-collection validation, and publishes both thecreate and touch events. The single source of truth for collection creation; REST and agentcallers both go through here."- [coll-data]+ [coll-data :- CreateCollectionArguments](u/prog1 (collections.db/insert-collection!(-> (apply-defaults-to-collection coll-data)write-check-authority-leveldiff --git a/src/metabase/collections/models/collection.clj b/src/metabase/collections/models/collection.cljindex 078d2c0ecd9..9c3ddb5331a 100644--- a/src/metabase/collections/models/collection.clj+++ b/src/metabase/collections/models/collection.clj@@ -111,12 +111,6 @@"Namespace for shared tenant collections":shared-tenant-collection)-(mu/defn shared-tenant-collection?- "Whether or not a collection is a tenant collection."- [{:keys [namespace]} :- [:or RootCollection [:map [:namespace {:optional true} [:maybe [:or :keyword :string]]]]]]- (= (some-> namespace name)- (name shared-tenant-ns)))-(defn trash-collection-id"The ID representing the Trash collection."[] (u/the-id (trash-collection)))@@ -333,7 +327,7 @@(mu/defmethod mi/can-read? :model/Collection([instance](or (is-trash? instance)- (perms/can-read-audit-helper :model/Collection instance)))+ (perms/can-read-audit-helper :model/Collection (t2/instance :model/Collection (select-keys instance [:id :namespace])))))([_model pk :- pos-int?](or (is-trash? pk)(mi/can-read? (collections.db/collection pk)))))@@ -393,11 +387,78 @@"Schema for a directory-style 'path' to the location of a Collection."[:fn #'valid-location-path?])+(def ^:private CollectionInstance+ "Schema for a Collection as it flows through this namespace: a raw `:model/Collection` Toucan instance, one+ hydrated with any of the computed keys this namespace or [[metabase.collections.children]] adds, or a+ null-padded row from the collection-items union query. Every key is optional because different call sites+ select or hydrate different subsets."+ [:map {:closed true}+ [:id {:optional true} ms/PositiveInt]+ [:name {:optional true} :string]+ [:description {:optional true} [:maybe :string]]+ [:archived {:optional true} :boolean]+ [:archived_directly {:optional true} [:maybe :boolean]]+ [:archive_operation_id {:optional true} [:maybe :string]]+ [:location {:optional true} LocationPath]+ [:personal_owner_id {:optional true} [:maybe ms/PositiveInt]]+ [:slug {:optional true} :string]+ [:namespace {:optional true} [:maybe [:or :keyword :string]]]+ [:authority_level {:optional true} [:maybe [:or :keyword :string]]]+ [:entity_id {:optional true} :string]+ [:created_at {:optional true} ms/TemporalInstant]+ [:type {:optional true} [:maybe [:or :keyword :string]]]+ [:is_sample {:optional true} :boolean]+ [:is_remote_synced {:optional true} [:maybe :boolean]]+ [:is_personal {:optional true} :boolean]+ [:is_upload {:optional true} [:maybe :boolean]]+ [:parent_id {:optional true} [:maybe ms/PositiveInt]]+ [:can_write {:optional true} :boolean]+ [:can_restore {:optional true} :boolean]+ [:can_delete {:optional true} :boolean]+ [:effective_location {:optional true} :string]+ [:effective_children {:optional true}+ [:maybe [:set [:map {:closed true}+ [:id ms/PositiveInt]+ [:name :string]+ [:description [:maybe :string]]+ [:type [:maybe :string]]]]]]+ [:collection_id {:optional true} [:maybe ms/PositiveInt]]+ [:collection_position {:optional true} [:maybe ms/PositiveInt]]+ [:collection_preview {:optional true} [:maybe :boolean]]+ [:collection_type {:optional true} [:maybe :string]]+ [:dashboard_id {:optional true} [:maybe ms/PositiveInt]]+ [:database_id {:optional true} [:maybe ms/PositiveInt]]+ [:dataset_query {:optional true} :nil]+ [:display {:optional true} [:maybe :string]]+ [:icon {:optional true} [:maybe :string]]+ [:last_edit_email {:optional true} [:maybe :string]]+ [:last_edit_first_name {:optional true} [:maybe :string]]+ [:last_edit_last_name {:optional true} [:maybe :string]]+ [:last_edit_timestamp {:optional true} [:maybe :string]]+ [:last_edit_user {:optional true} [:maybe :int]]+ [:last_used_at {:optional true} [:maybe ms/TemporalInstant]]+ [:model {:optional true} [:maybe :string]]+ [:model_ranking {:optional true} [:maybe :int]]+ [:moderated_status {:optional true} [:maybe :string]]+ [:query_type {:optional true} [:maybe :string]]+ [:table_id {:optional true} [:maybe ms/PositiveInt]]+ [:total_count {:optional true} [:maybe :int]]+ [:here {:optional true} [:set :keyword]]+ [:below {:optional true} [:set :keyword]]+ [:is_library_root {:optional true} :boolean]+ [:effective_ancestors {:optional true} [:sequential ::collections.schema/collection-or-root]]])++(mu/defn shared-tenant-collection?+ "Whether or not a collection is a tenant collection."+ [{:keys [namespace]} :- [:or RootCollection CollectionInstance]]+ (= (some-> namespace name)+ (name shared-tenant-ns)))+(mu/defn location-path :- LocationPath"Build a 'location path' from a sequence of `collections-or-ids`.(location-path 10 20) ; -> \"/10/20/\""- [& collections-or-ids :- [:* [:or ms/PositiveInt :map]]]+ [& collections-or-ids :- [:* [:or ms/PositiveInt CollectionInstance]]](if-not (seq collections-or-ids)"/"(str@@ -503,17 +564,14 @@(def ^:private CollectionWithLocationOrRoot[:orRootCollection- [:map- [:location LocationPath]]])+ CollectionInstance])(def CollectionWithLocationAndIDOrRoot"Schema for a valid `CollectionInstance` that has valid `:location` and `:id` properties, or the special`root-collection` placeholder object."[:orRootCollection- [:map- [:location LocationPath]- [:id ms/PositiveInt]]])+ CollectionInstance])(mu/defn- parent :- CollectionWithLocationAndIDOrRoot"Fetch the parent Collection of `collection`, or the Root Collection special placeholder object if this is a@@ -553,8 +611,10 @@Practically, use `user-or-site` = `:site` when insert or update the name in database,and `:user` when we need the name for displaying purposes"- [first-name last-name email user-or-site]- {:pre [(#{:user :site} user-or-site)]}+ [first-name :- [:maybe :string]+ last-name :- [:maybe :string]+ email :- [:maybe :string]+ user-or-site :- [:enum :user :site]](if (= :user user-or-site)(cond(and first-name last-name) (tru "{0} {1}''s Personal Collection" first-name last-name)@@ -566,14 +626,16 @@(mu/defn user->personal-collection-names :- [:map-of ms/PositiveInt :string]"Come up with a nice name for the Personal Collection for the passed `user-or-ids`.Returns a map of user-id -> name"- [user-or-ids user-or-site]+ [user-or-ids :- [:sequential ms/PositiveInt]+ user-or-site :- [:enum :user :site]](into {} (when-let [ids (seq (filter some? (map u/the-id user-or-ids)))](update-vals (collections.db/user-name-parts-by-id ids)#(format-personal-collection-name (:first_name %) (:last_name %) (:email %) user-or-site)))))(mu/defn user->personal-collection-name :- ms/NonBlankString"Calls `user->personal-collection-names` for a single user-id and returns the name"- [user-or-id user-or-site]+ [user-or-id :- ms/PositiveInt+ user-or-site :- [:enum :user :site]](first (vals (user->personal-collection-names [user-or-id] user-or-site))))(defn personal-collections-with-ui-details@@ -620,9 +682,7 @@(def ^:private CollectionWithLocationAndPersonalOwnerID"Schema for a Collection instance that has a valid `:location`, and a `:personal_owner_id` key *present* (but notnecessarily non-nil)."- [:map- [:location LocationPath]- [:personal_owner_id [:maybe ms/PositiveInt]]])+ CollectionInstance)(mu/defn is-personal-collection-or-descendant-of-one? :- :boolean"Is `collection` a Personal Collection, or a descendant of one?"@@ -639,8 +699,7 @@(def ^:private CollectionWithNamespace"Schema for a Collection instance that has a valid `:location`, and a `:namespace` key *present* (but notnecessarily non-nil)."- [:map- [:namespace [:maybe [:or :keyword :string]]]])+ CollectionInstance)(mu/defn is-dedicated-tenant-collection-or-descendant? :- :boolean"Is `collection` a Tenant Collection, or a descendant of one?"@@ -648,17 +707,21 @@;; If collection has namespace = "tenant-specific" we know it's in the dedicated tenant namespace(= (some-> (:namespace collection) name) "tenant-specific"))+(def ^:private UserOrId+ "A User ID, or anything with one (e.g. a full User instance) — [[u/the-id]] accepts either."+ [:or ms/PositiveInt :metabase.users.schema/user])+(mu/defn user->existing-personal-collection :- [:maybe (ms/InstanceOf :model/Collection)]"For a `user-or-id`, return their personal Collection, if it already exists.Use [[metabase.collections.models.collection/user->personal-collection]] to fetch their personal Collection *and*create it if needed."- [user-or-id]+ [user-or-id :- UserOrId](collections.db/personal-collection-of-user (u/the-id user-or-id)))(mu/defn user->personal-collection :- [:maybe (ms/InstanceOf :model/Collection)]"Return the Personal Collection for `user-or-id`, if it already exists; if not, create it and return it.Personal collection should be created on user creation, but creates if missing for backwards compatibility"- [user-or-id]+ [user-or-id :- UserOrId];; API key users do not get personal collections(when-not (api-key/is-api-key-user? (u/the-id user-or-id))(or (user->existing-personal-collection user-or-id)@@ -696,7 +759,7 @@of that Collection. Exists because this needs to be known to calculate the Current User's permissions set, which isdone for every API call; this function is an attempt to make fetching this information as efficient as reasonablypossible."- [user-or-id]+ [user-or-id :- ms/PositiveInt](into [](when-let [personal-collection-id (user->personal-collection-id (u/the-id user-or-id))](conj@@ -767,7 +830,7 @@;; breadcrumbing in the frontend.(def ^:private CollectionVisibilityConfig- [:map+ [:map {:closed true}[:cte-name {:optional true} [:maybe :keyword]][:include-trash-collection? {:optional true} :boolean][:include-archived-items {:optional true} [:enum :only :exclude :all]]@@ -776,7 +839,7 @@[:effective-child-of {:optional true} [:maybe CollectionWithLocationAndIDOrRoot]]])(def ^:private UserScope- [:map+ [:map {:closed true}[:current-user-id pos-int?][:is-superuser? :boolean]])@@ -1045,7 +1108,7 @@;;; +----------------------------------------------------------------------------------------------------------------+(mu/defn- ancestors* :- [:maybe [:sequential (ms/InstanceOf :model/Collection)]]- [{:keys [location]}]+ [{:keys [location]} :- CollectionWithLocationOrRoot](when-let [ancestor-ids (seq (location-path->ids location))](collections.db/ancestor-summaries ancestor-ids)))@@ -1059,7 +1122,7 @@(mu/defn- effective-ancestors*"Given a collection, return the effective ancestors of that collection."[collection :- [:maybe CollectionWithLocationOrRoot]- collection-id->collection :- :map]+ collection-id->collection :- [:map-of :metabase.lib.schema.id/collection ::collections.schema/collection]](if (or (nil? collection)(collection.root/is-root-collection? collection))[]@@ -1176,7 +1239,10 @@(mu/defn- effective-children-where-clause"Given a collection, return the `WHERE` clause appropriate to return all the collections we want to show as itseffective children."- [collection collection-table-alias visibility-config & additional-honeysql-where-clauses]+ [collection :- CollectionWithLocationAndIDOrRoot+ collection-table-alias :- :keyword+ visibility-config :- CollectionVisibilityConfig+ & additional-honeysql-where-clauses :- [:* ::h2x/honeysql-expr]](into[:and(effective-child-of-filter-clause collection collection-table-alias visibility-config)@@ -1214,7 +1280,7 @@access, but that are children of Collections they cannot access; in the example above, E and F are such nodes."[collection :- CollectionWithLocationAndIDOrRootvisibility-config :- CollectionVisibilityConfig- & additional-honeysql-where-clauses]+ & additional-honeysql-where-clauses :- [:* ::h2x/honeysql-expr]]^:allow-subquery {:select [:id :name :description :type]:from [[:collection :col]]:where (apply effective-children-where-clause collection :col visibility-config additional-honeysql-where-clauses)})@@ -1457,10 +1523,10 @@(let [{collection-after-update :collection_id :as model-after-update} <>](when (remote-synced-collection? collection-after-update)(check-non-remote-synced-dependencies model-after-update)- (when (and (api/column-will-change? :archived model-before-update model-after-update)+ (when (and (api/column-will-change? (:archived model-before-update) (get model-after-update :archived ::api/not-provided))(:archived model-after-update))(check-remote-synced-dependents model-after-update)))- (when (and (api/column-will-change? :collection_id model-before-update model-after-update)+ (when (and (api/column-will-change? (:collection_id model-before-update) (get model-after-update :collection_id ::api/not-provided))(moving-from-remote-synced? collection-before-update collection-after-update))(check-remote-synced-dependents model-after-update)))))@@ -1511,7 +1577,7 @@(for [collection-or-id (conscollection(collections.db/unarchived-collection-ids-with-location-like (str (children-location collection) "%")))]- (perms/collection-readwrite-path collection-or-id))))+ (perms/collection-readwrite-path (u/the-id collection-or-id)))))(mu/defn perms-for-archiving :- [:set perms/PathSchema]"Return the set of Permissions needed to archive or unarchive a `collection`. Since archiving a Collection is@@ -1580,7 +1646,7 @@(str (children-location collection) "%")archive-operation-id)](set- (cons (perms/collection-readwrite-path collection)+ (cons (perms/collection-readwrite-path (u/the-id collection))(map perms/collection-readwrite-path descendant-ids)))))(def ^:dynamic *allow-modifying-tenant-root-collections?*@@ -1642,7 +1708,9 @@[collection :- CollectionWithLocationAndIDOrRoot;; `updates` is a map *possibly* containing `parent_id`. This allows us to distinguish;; between specifying a `nil` parent_id (move to the root) and not specifying a parent_id.- updates :- [:map [:parent_id {:optional true} [:maybe ms/PositiveInt]]]]+ updates :- [:map {:closed true}+ [:parent_id {:optional true} [:maybe ms/PositiveInt]]+ [:archived {:optional true} :boolean]]](assert (:archive_operation_id collection))(let [archive-operation-id (:archive_operation_id collection)current-parent-id (:parent_id (t2/hydrate collection :parent_id))@@ -1697,8 +1765,9 @@[collection :- CollectionWithLocationAndIDOrRoot;; `updates` is a map *possibly* containing `parent_id`. This allows us to distinguish;; between specifying a `nil` parent_id (move to the root) and not specifying a parent_id.- updates :- [:map [:parent_id {:optional true} [:maybe ms/PositiveInt]- :archived :boolean]]]+ updates :- [:map {:closed true}+ [:parent_id {:optional true} [:maybe ms/PositiveInt]]+ [:archived {:optional true} :boolean]]](if (:archived updates)(archive-collection! collection)(unarchive-collection! collection updates)))@@ -1707,7 +1776,7 @@"Move a Collection and all its descendant Collections from its current `location` to a `new-location`."[collection :- CollectionWithLocationAndIDOrRootnew-location :- LocationPath- & [into-remote-synced? :- :boolean]]+ & [into-remote-synced?] :- [:* :boolean]](let [orig-children-location (children-location collection)new-children-location (children-location (assoc collection :location new-location))will-be-in-trash? (str/starts-with? new-location (trash-path))@@ -1818,7 +1887,7 @@allowed. Personal Collections and TCs have lots of restrictions -- you can't archive them, for example, nor can youtransfer them to other Users."[collection-before-updates :- CollectionWithLocationAndIDOrRoot- collection-updates :- :map]+ collection-updates :- ::collections.schema/collection.update];; you're not allowed to change the `:personal_owner_id` of a Collection!;; double-check and make sure it's not just the existing value getting passed back in for whatever reason(let [ctype (if (:personal_owner_id collection-before-updates)@@ -1833,7 +1902,8 @@:archived (tru "You cannot archive a {0}." ctype)}](when-let [[k msg] (->> unchangeable(filter (fn [[k _msg]]- (api/column-will-change? k collection-before-updates collection-updates)))+ (api/column-will-change? (get collection-before-updates k)+ (get collection-updates k ::api/not-provided))))first)](throw(ex-info msg {:status-code 400 :errors {k msg}})))))@@ -1863,7 +1933,7 @@bad experience -- we do not want a User to move a Collection that they have read/write perms for (by definition) tosomewhere else and lose all access for it."[collection :- (ms/InstanceOf :model/Collection) new-location :- LocationPath]- (copy-collection-permissions! (parent {:location new-location}) (cons collection (descendants collection))))+ (copy-collection-permissions! (parent {:location new-location}) (map u/the-id (cons collection (descendants collection)))))(mu/defn- revoke-perms-when-moving-into-personal-collection!"When moving a `collection` that is *not* a descendant of a Personal Collection into a Personal Collection or one of@@ -1875,7 +1945,7 @@(collections.db/delete-permissions-with-objects! (for [collection (cons collection (descendants collection))path-fn [perms/collection-read-pathperms/collection-readwrite-path]]- (path-fn collection))))+ (path-fn (u/the-id collection)))))(defn- update-perms-when-moving-across-personal-boundry!"If a Collection is moving 'across the boundry' and will become a descendant of a Personal Collection, or will cease@@ -1959,7 +2029,7 @@(check-library-update collection);; (4) If we're moving a Collection from a location on a Personal Collection hierarchy to a location not on one,;; or vice versa, we need to grant/revoke permissions as appropriate (see above for more details)- (when (api/column-will-change? :location collection-before-updates collection-updates)+ (when (api/column-will-change? (:location collection-before-updates) (get collection-updates :location ::api/not-provided))(update-perms-when-moving-across-personal-boundry! collection-before-updates collection-updates);; (4.5) If we're moving a Collection across the tenant-specific namespace boundary, we need to adjust;; permissions accordingly (delete when moving in, grant when moving out)@@ -2003,8 +2073,8 @@(when (:personal_owner_id collection)(throw (Exception. (tru "You cannot delete a Personal Collection!")))));; Delete permissions records for this Collection- (collections.db/delete-permissions-with-objects! [(perms/collection-readwrite-path collection)- (perms/collection-read-path collection)]))+ (collections.db/delete-permissions-with-objects! [(perms/collection-readwrite-path (u/the-id collection))+ (perms/collection-read-path (u/the-id collection))]));;; -------------------------------------------------- IModel Impl ---------------------------------------------------@@ -2020,8 +2090,8 @@;; This is not entirely accurate as you need to be a superuser to modify a collection itself (e.g., changing its;; name) but if you have write perms you can add/remove cards#{(case read-or-write- :read (perms/collection-read-path collection-or-id)- :write (perms/collection-readwrite-path collection-or-id))})))+ :read (perms/collection-read-path (u/the-id collection-or-id))+ :write (perms/collection-readwrite-path (u/the-id collection-or-id)))})))(def instance-analytics-collection-type"The value of the `:type` field for the `instance-analytics` Collection created in [[metabase-enterprise.audit-app.audit]]"@@ -2153,7 +2223,7 @@(check-allowed-to-change-collection (t2/select-one Card :id 100) http-request-body)"[object-before-update object-updates];; if collection_id is set to change...- (when (api/column-will-change? :collection_id object-before-update object-updates)+ (when (api/column-will-change? (:collection_id object-before-update) (get object-updates :collection_id ::api/not-provided));; check that we're allowed to modify the old Collection(if-let [coll-id (:collection_id object-before-update)](api/write-check :model/Collection coll-id)@@ -2329,7 +2399,7 @@archived-directly? (:archived_directly coll)parent-archived? (get parent-id->archived? parent-id false)descendant-ids (get op-id->descendant-ids (:archive_operation_id coll) [])- perm-paths (set (cons (perms/collection-readwrite-path coll)+ perm-paths (set (cons (perms/collection-readwrite-path (u/the-id coll))(map perms/collection-readwrite-path descendant-ids)))]](assoc coll :can_restore (boolean (and (:archived coll)archived-directly?diff --git a/src/metabase/collections/schema.clj b/src/metabase/collections/schema.cljindex 4a3863c4aa9..43638984d68 100644--- a/src/metabase/collections/schema.clj+++ b/src/metabase/collections/schema.clj@@ -100,23 +100,36 @@(mr/def ::collection"A Collection as selected from the app DB: every column of `:collection`."+ [:merge+ ::collection.update+ [:map {:closed true}+ [:id ::lib.schema.id/collection]+ [:can_write {:optional true} :boolean]+ [:is_personal {:optional true} [:maybe :boolean]]+ [:effective_location {:optional true} [:maybe :string]]]])++(mr/def ::root-collection+ "The placeholder for the Root Collection, which has no row, as `metabase.collections.models.collection.root` builds it."[:map {:closed true}- [:id ::lib.schema.id/collection]- [:name :string]- [:description [:maybe :string]]- [:archived :boolean]- [:location :string]- [:personal_owner_id [:maybe ::lib.schema.id/user]]- [:slug :string]- [:namespace [:maybe [:or :keyword :string]]]- [:authority_level [:maybe [:or :keyword :string]]]- [:entity_id :string]- [:created_at ms/TemporalInstant]- [:type [:maybe [:or :keyword :string]]]- [:is_sample :boolean]- [:archive_operation_id [:maybe :string]]- [:archived_directly [:maybe :boolean]]- [:is_remote_synced [:maybe :boolean]]])+ [:metabase.collections.models.collection.root/is-root? [:= true]]+ [:authority_level {:optional true} [:maybe [:or :keyword :string]]]+ [:id {:optional true} [:= "root"]]+ [:name {:optional true} :string]+ [:namespace {:optional true} [:maybe [:or :keyword :string]]]+ [:is_personal {:optional true} :boolean]+ [:is_remote_synced {:optional true} :boolean]+ [:can_write {:optional true} :boolean]+ [:parent_id {:optional true} :nil]+ [:effective_location {:optional true} [:maybe :string]]+ [:effective_ancestors {:optional true} [:sequential [:ref ::collection]]]+ [:can_restore {:optional true} :boolean]+ [:can_delete {:optional true} :boolean]])++(mr/def ::collection-or-root+ "A Collection row, or the Root Collection placeholder."+ [:multi {:dispatch (fn [x] (if (:metabase.collections.models.collection.root/is-root? x) :root :row))}+ [:root [:ref ::root-collection]]+ [:row [:ref ::collection]]])(mr/def ::collection.update"What an update (or insert) of a Collection accepts: every column of `:collection` except `id`, all optional."diff --git a/src/metabase/collections/update.clj b/src/metabase/collections/update.cljindex efea475395f..6041c7d983d 100644--- a/src/metabase/collections/update.clj+++ b/src/metabase/collections/update.clj@@ -28,7 +28,7 @@are left alone; `:archived` and `:parent_id` are acted on only when present, so a caller thatwants the REST endpoint's \"omitted means unarchived\" semantics has to supply that defaultitself."- [:map+ [:map {:closed true}[:name {:optional true} [:maybe ms/NonBlankString]][:description {:optional true} [:maybe ms/NonBlankString]][:archived {:optional true} [:maybe :boolean]]@@ -40,7 +40,7 @@which will not cause the archive notification code to fire. This will delete the relevant alerts and notify theusers just as if they had be archived individually via the card API."[& {:keys [collection-before-update collection-updates actor]}]- (when (api/column-will-change? :archived collection-before-update collection-updates)+ (when (api/column-will-change? (:archived collection-before-update) (get collection-updates :archived ::api/not-provided))(doseq [card (collections.db/cards-in-collection (u/the-id collection-before-update))](notification.card/delete-card-notifications-and-notify! :event/card-update.notification-deleted.card-archived actor card))))@@ -76,7 +76,7 @@then move it to the trash."[collection-before-update collection-updates];; sanity check- (when (api/column-will-change? :archived collection-before-update collection-updates)+ (when (api/column-will-change? (:archived collection-before-update) (get collection-updates :archived ::api/not-provided))(collection/archive-or-unarchive-collection!collection-before-update(select-keys collection-updates [:parent_id :archived]))@@ -88,10 +88,14 @@"If `collection-updates` specifies that we should either move or archive the collection (archiving means 'moving tothe trash' so it makes sense to deal with them together), do the appropriate permissions checks and changes."[collection-before-update collection-updates]- (condp #(api/column-will-change? %1 collection-before-update %2) collection-updates- :archived (archive-collection! collection-before-update collection-updates)- :parent_id (move-collection! collection-before-update collection-updates)- :no-op))+ (cond+ (api/column-will-change? (:archived collection-before-update) (get collection-updates :archived ::api/not-provided))+ (archive-collection! collection-before-update collection-updates)++ (api/column-will-change? (:parent_id collection-before-update) (get collection-updates :parent_id ::api/not-provided))+ (move-collection! collection-before-update collection-updates)++ :else :no-op))(mu/defn update-collection!"Modify the collection with `id`, including archiving or unarchiving it, or moving it. Write-checks the collection,diff --git a/src/metabase/collections_rest/api.clj b/src/metabase/collections_rest/api.cljindex 152b303c05b..ce6f0bb83fb 100644--- a/src/metabase/collections_rest/api.clj+++ b/src/metabase/collections_rest/api.clj@@ -6,7 +6,6 @@(:require[malli.core :as mc][malli.transform :as mtx]- [malli.util][metabase.api.common :as api][metabase.api.macros :as api.macros][metabase.collections-rest.db :as collections-rest.db]@@ -21,6 +20,7 @@[metabase.permissions.core :as perms][metabase.premium-features.core :as premium-features][metabase.queries.core :as queries]+ [metabase.queries.schema :as queries.schema][metabase.request.core :as request][metabase.util :as u][metabase.util.i18n :as i18n]@@ -254,7 +254,7 @@(mu/defn- dashboard-question-candidates"Implementation for the `dashboard-question-candidates` endpoints."- [collection-id]+ [collection-id :- [:maybe :metabase.lib.schema.id/collection]](api/check-403 api/*is-superuser?*)(let [all-cards-in-collection (t2/hydrate (collections-rest.db/top-level-cards-in-collection collection-id) :in_dashboards)](filter@@ -268,14 +268,24 @@(-> card :in_dashboards first :collection_id))))all-cards-in-collection)))+(def ^:private CardInDashboard+ [:map {:closed true}+ [:name :string]+ [:collection_id [:maybe :metabase.lib.schema.id/collection]]+ [:description [:maybe :string]]+ [:id :metabase.lib.schema.id/dashboard]+ [:archived :boolean]+ [:enable_embedding :boolean]])+(mu/defn- present-dashboard-question-candidate- [{:keys [in_dashboards] :as card}]+ [{:keys [in_dashboards] :as card}+ :- [:merge ::queries.schema/card [:map {:closed true} [:in_dashboards [:sequential CardInDashboard]]]]](-> card(select-keys [:id :name :description])(assoc :sole_dashboard_info (-> in_dashboards first (select-keys [:id :name :description])))))(mu/defn- present-dashboard-question-candidates- [cards]+ [cards :- [:sequential [:merge ::queries.schema/card [:map {:closed true} [:in_dashboards [:sequential CardInDashboard]]]]]];; we're paginating in Clojure rather than in the query itself because the criteria here is quite complicated to;; express in SQL: we need to join to `report_dashboardcard` AND `dashboardcard_series`, and find cards that have;; exactly one matching dashboard across both of those joins. I'm sure it's doable, but for now we can just do thisdiff --git a/src/metabase/comments/api.clj b/src/metabase/comments/api.cljindex 62855feb7c1..c79ce0d57d6 100644--- a/src/metabase/comments/api.clj+++ b/src/metabase/comments/api.clj@@ -13,7 +13,6 @@[metabase.comments.render :as comments.render][metabase.comments.schema :as comments.schema][metabase.events.core :as events]- [metabase.lib.schema.literal :as lib.schema.literal][metabase.models.interface :as mi][metabase.request.core :as request][metabase.util :as u]@@ -63,28 +62,13 @@::comments.schema/prose-mirror-node](deferred-tru "Comment content must be valid JSON.")))-(def ^:private CommentHighlight- "The chart point a comment is anchored to. Identity only — which column, and which dimension values- pick out the point — so the client can re-find it in a result set it is separately authorized to- read."- [:map {:closed true}- [:columnName {:optional true} [:maybe :string]]- [:dimensions {:optional true}- [:maybe [:sequential [:map {:closed true}- [:columnName {:optional true} [:maybe :string]]- [:value {:optional true} [:ref ::lib.schema.literal/literal]]]]]]])-(def CommentContext"Context stored alongside a comment"(mu/with-api-error-message[:and{:error/message "Comment context must be a valid JSON object":json-schema {:type "object"}}- [:map {:closed true}- [:timeline_id {:optional true} [:maybe ms/PositiveInt]]- [:exploration_query_ids {:optional true} [:maybe [:sequential ms/PositiveInt]]]- [:highlighted {:optional true} [:maybe CommentHighlight]]- [:highlight_label {:optional true} [:maybe [:string {:max 1000}]]]]]+ ::comments.schema/comment.context](deferred-tru "Comment context must be a valid JSON object.")))(def CreateCommentdiff --git a/src/metabase/comments/schema.clj b/src/metabase/comments/schema.cljindex e56f5bd1e40..ba4dd2dfe54 100644--- a/src/metabase/comments/schema.clj+++ b/src/metabase/comments/schema.clj@@ -3,6 +3,7 @@(:require[metabase.lib.core :as lib][metabase.lib.schema.id :as lib.schema.id]+ [metabase.lib.schema.literal :as lib.schema.literal][metabase.util.malli :as mu][metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms]))@@ -32,31 +33,50 @@"The `:content` column of a Comment, decoded: the ProseMirror document the comment was written in."::prose-mirror-node)+(mr/def ::prose-mirror-node.json-attrs+ "The `attrs` of a ProseMirror node as decoded JSON, before normalization stringifies its keys: the keys belong to the+ editor's node type and may be keywords or strings."+ [:map {:closed false, ::mr/deliberately-open true, :description "ProseMirror node attrs"}])++(mr/def ::prose-mirror-node.json+ "A ProseMirror node as decoded JSON, before normalization stringifies the keys of its `attrs`."+ [:map {:closed true}+ [:type :string]+ [:attrs {:optional true} [:maybe [:ref ::prose-mirror-node.json-attrs]]]+ [:content {:optional true} [:sequential [:ref ::prose-mirror-node.json]]]+ [:marks {:optional true} [:sequential [:ref ::prose-mirror-node.json]]]+ [:text {:optional true} :string]])+(mu/defn normalize-content :- [:maybe ::comment.content]"Normalize a comment's content on its way in from the API or out of the application database."- [content]+ [content :- [:maybe ::prose-mirror-node.json]](some->> content (lib/normalize ::comment.content)))+(mr/def ::comment.highlight+ "The chart point a comment is anchored to. Identity only — which column, and which dimension values+ pick out the point — so the client can re-find it in a result set it is separately authorized to+ read."+ [:map {:closed true}+ [:columnName {:optional true} [:maybe :string]]+ [:dimensions {:optional true}+ [:maybe [:sequential [:map {:closed true}+ [:columnName {:optional true} [:maybe :string]]+ [:value {:optional true} [:ref ::lib.schema.literal/literal]]]]]]])+(mr/def ::comment.context"The `:context` column of a Comment, decoded."- :map)+ [:map {:closed true}+ [:timeline_id {:optional true} [:maybe ms/PositiveInt]]+ [:exploration_query_ids {:optional true} [:maybe [:sequential ms/PositiveInt]]]+ [:highlighted {:optional true} [:maybe ::comment.highlight]]+ [:highlight_label {:optional true} [:maybe [:string {:max 1000}]]]])(mr/def ::comment"A Comment as selected from the app DB: every column of `:comment`."- [:map {:closed true}- [:id ms/PositiveInt]- [:parent_comment_id [:maybe ms/PositiveInt]]- [:target_type [:or :keyword :string]]- [:target_id ms/PositiveInt]- [:child_target_id [:maybe :string]]- [:creator_id ::lib.schema.id/user]- [:content ::comment.content]- [:is_resolved :boolean]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]- [:deleted_at [:maybe ms/TemporalInstant]]- [:content_html [:maybe :string]]- [:context [:maybe ::comment.context]]])+ [:merge+ ::comment.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::comment.update"What an update (or insert) of a Comment accepts: every column of `:comment` except `id`, all optional."@@ -76,12 +96,10 @@(mr/def ::comment-reaction"A CommentReaction as selected from the app DB: every column of `:comment_reaction`."- [:map {:closed true}- [:id ms/PositiveInt]- [:comment_id ms/PositiveInt]- [:user_id ::lib.schema.id/user]- [:emoji :string]- [:created_at ms/TemporalInstant]])+ [:merge+ ::comment-reaction.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::comment-reaction.update"What an update (or insert) of a CommentReaction accepts: every column of `:comment_reaction` except `id`, all optional."diff --git a/src/metabase/content_translation/schema.clj b/src/metabase/content_translation/schema.cljindex edc2455711f..c08c02b43b1 100644--- a/src/metabase/content_translation/schema.clj+++ b/src/metabase/content_translation/schema.clj@@ -6,11 +6,10 @@(mr/def ::content-translation"A ContentTranslation as selected from the app DB: every column of `:content_translation`."- [:map {:closed true}- [:id ms/PositiveInt]- [:locale :string]- [:msgid :string]- [:msgstr :string]])+ [:merge+ ::content-translation.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::content-translation.update"What an update (or insert) of a ContentTranslation accepts: every column of `:content_translation` except `id`, all optional."diff --git a/src/metabase/content_verification/models/moderation_review.clj b/src/metabase/content_verification/models/moderation_review.cljindex b9b023b90dc..5a27915cb8d 100644--- a/src/metabase/content_verification/models/moderation_review.clj+++ b/src/metabase/content_verification/models/moderation_review.clj@@ -60,7 +60,7 @@(mu/defn create-review!"Create a new ModerationReview"[params :-- [:map+ [:map {:closed true}[:moderated_item_id ms/PositiveInt][:moderated_item_type moderation/moderated-item-types][:moderator_id ms/PositiveInt]diff --git a/src/metabase/content_verification/schema.clj b/src/metabase/content_verification/schema.cljindex 5ee67b92433..7a0ba9be44e 100644--- a/src/metabase/content_verification/schema.clj+++ b/src/metabase/content_verification/schema.clj@@ -1,21 +1,17 @@(ns metabase.content-verification.schema"Malli schemas for the content-verification module."(:require+ [metabase.users.schema][metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms]))(mr/def ::moderation-review"A ModerationReview as selected from the app DB: every column of `:moderation_review`."- [:map {:closed true}- [:id ms/PositiveInt]- [:updated_at ms/TemporalInstant]- [:created_at ms/TemporalInstant]- [:status [:maybe [:or :keyword :string]]]- [:text [:maybe :string]]- [:moderated_item_id ms/PositiveInt]- [:moderated_item_type [:or :keyword :string]]- [:moderator_id ms/PositiveInt]- [:most_recent :boolean]])+ [:merge+ ::moderation-review.update+ [:map {:closed true}+ [:id ms/PositiveInt]+ [:user {:optional true} [:maybe :metabase.users.schema/user]]]])(mr/def ::moderation-review.update"What an update (or insert) of a ModerationReview accepts: every column of `:moderation_review` except `id`, all optional."diff --git a/src/metabase/contextual_interestingness/llm.clj b/src/metabase/contextual_interestingness/llm.cljindex 1b444ae3440..f80aa62d50c 100644--- a/src/metabase/contextual_interestingness/llm.clj+++ b/src/metabase/contextual_interestingness/llm.clj@@ -24,7 +24,7 @@[generate-metric?]{:type "object":additionalProperties false- :properties (cond-> {:chart_description+ :properties (cond-> {"chart_description"{:type "string":description (str "One short sentence (≤ 25 words) describing WHAT THE CHART IS — the metric and the dimension it's broken out by, in the terms the user would use. ""If a Slicing line is present (segment filter, top-N + Other rollup, specific-values subset, per-value over-time view), name it — it's part of what the chart is. "@@ -32,17 +32,17 @@"Ground filters, joins, and aggregation in the COMPILED SQL block when one is provided — do not restate the title. ""Write this FIRST so the rest of the response is grounded in it.")}}generate-metric?- (assoc :metric_description+ (assoc "metric_description"{:type "string":description (str "One short sentence (≤ 25 words) describing the metric in business terms (what it measures, what's included/excluded). ""Read the aggregation, joins, and WHERE clause in the COMPILED SQL block (when provided) to identify what's being summed/counted/averaged and which subset of rows. ""Do not restate the chart name.")}):always- (assoc :reasoning+ (assoc "reasoning"{:type "string":description "One short sentence justifying the score, retained for debugging only."}- :score+ "score"{:type "number":minimum 0:maximum 1diff --git a/src/metabase/dashboards/db.clj b/src/metabase/dashboards/db.cljindex 26463a001ec..76f8ff78f0a 100644--- a/src/metabase/dashboards/db.clj+++ b/src/metabase/dashboards/db.clj@@ -277,7 +277,7 @@(mu/defn ensure-integer-link-card-id"Return `id` if it is an integer, else throw a 400."- [id :- :some]+ [id :- ms/VisualizationSettingsValue](when-not (integer? id)(throw (ex-info "Link card entity id must be an integer"{:status-code 400, :id id})))diff --git a/src/metabase/dashboards/models/dashboard.clj b/src/metabase/dashboards/models/dashboard.cljindex 08d568cb202..b194b259550 100644--- a/src/metabase/dashboards/models/dashboard.clj+++ b/src/metabase/dashboards/models/dashboard.clj@@ -22,6 +22,7 @@[metabase.permissions.core :as perms][metabase.public-sharing.core :as public-sharing][metabase.queries.core :as queries]+ [metabase.queries.schema :as queries.schema][metabase.query-permissions.core :as query-perms][metabase.query-processor.metadata :as qp.metadata][metabase.search.core :as search]@@ -198,11 +199,11 @@don't drift. Call inside the same transaction as the dashboard update itself."[current-dash updates](let [id (:id current-dash)]- (when (api/column-will-change? :archived current-dash updates)+ (when (api/column-will-change? (:archived current-dash) (get updates :archived ::api/not-provided))(if (:archived updates)(dashboards.db/archive-dashboard-questions! id)(dashboards.db/unarchive-dashboard-questions! id)))- (when (api/column-will-change? :collection_id current-dash updates)+ (when (api/column-will-change? (:collection_id current-dash) (get updates :collection_id ::api/not-provided))(dashboards.db/move-dashboard-questions! id (:collection_id updates)))));;; +----------------------------------------------------------------------------------------------------------------+@@ -242,21 +243,19 @@(let [new-param-field-ids (dashboard-id->param-field-ids dashboard-or-id)](update-field-values-for-on-demand-dbs! old-param-field-ids new-param-field-ids)))))-(def ^:private DashboardWithSeriesAndCard- [:map- [:id ms/PositiveInt]- [:dashcards [:sequential [:map- [:card_id {:optional true} [:maybe ms/PositiveInt]]- [:card {:optional true} [:maybe [:map- [:id ms/PositiveInt]]]]]]]])-(mu/defn update-dashcards!"Update the `dashcards` belonging to `dashboard`.This function is provided as a convenience instead of doing this yourself; it also makes sure various cleanup stepsare performed when finished, for example updating FieldValues for On-Demand DBs.Returns `nil`."- [dashboard :- DashboardWithSeriesAndCard- new-dashcards :- [:sequential :map]]+ [dashboard :- ::dashboards.schema/dashboard+ new-dashcards :- [:sequential [:merge+ ::dashboards.schema/dashboard-card.update+ [:map {:closed true}+ [:id ms/PositiveInt]+ [:series {:optional true} [:maybe [:sequential [:map {:closed true} [:id ms/PositiveInt]]]]]+ [:card {:optional true} [:maybe ::queries.schema/card]]+ [:collection_authority_level {:optional true} [:maybe [:or :keyword :string]]]]]]](let [old-dashcards (:dashcards dashboard)id->old-dashcard (m/index-by :id old-dashcards)old-dashcard-ids (set (keys id->old-dashcard))@@ -330,20 +329,22 @@(defn save-transient-dashboard!"Save a denormalized description of `dashboard`."[dashboard parent-collection-id]- (queries/check-parameter-source-card-permissions (:parameters dashboard))+ (queries/check-parameter-source-card-permissions+ (lib/normalize [:maybe [:sequential :metabase.parameters.schema/parameter]] (:parameters dashboard)))(t2/with-transaction [_conn](let [{dashcards :dashcardstabs :tabs:keys [description] :as dashboard} (i18n/localized-strings->strings dashboard)dashboard (dashboards.db/insert-dashboard!- (-> dashboard- (dissoc :dashcards :tabs :rule :related- :transient_name :transient_filters :param_fields :more- :public_uuid :made_public_by_id- :enable_embedding :embedding_params)- (assoc :description description- :collection_id parent-collection-id- :creator_id api/*current-user-id*)))+ (->> (-> dashboard+ (dissoc :dashcards :tabs :rule :related+ :transient_name :transient_filters :param_fields :more+ :public_uuid :made_public_by_id+ :enable_embedding :embedding_params)+ (assoc :description description+ :collection_id parent-collection-id+ :creator_id api/*current-user-id*))+ (lib/normalize ::dashboards.schema/dashboard.update))){:keys [old->new-tab-id]} (dashboard-tab/do-update-tabs! (:id dashboard) nil tabs)dashcards-to-add (for [dashcard dashcards](let [card (some-> dashcard :carddiff --git a/src/metabase/dashboards/models/dashboard_card.clj b/src/metabase/dashboards/models/dashboard_card.cljindex 189f7eaffa7..ed6d7350149 100644--- a/src/metabase/dashboards/models/dashboard_card.clj+++ b/src/metabase/dashboards/models/dashboard_card.clj@@ -3,9 +3,13 @@[clojure.set :as set][medley.core :as m][metabase.dashboards.db :as dashboards.db]+ [metabase.dashboards.schema :as dashboards.schema]+ [metabase.lib.core :as lib][metabase.models.interface :as mi][metabase.models.serialization :as serdes][metabase.parameters.core :as parameters]+ [metabase.parameters.schema :as parameters.schema]+ [metabase.queries.schema :as queries.schema][metabase.util :as u][metabase.util.malli :as mu][metabase.util.malli.schema :as ms]@@ -106,7 +110,7 @@[dashboard-card](t2/instance :model/DashboardCard(-> dashboard-card- (m/update-existing :parameter_mappings parameters/normalize-parameter-mappings)+ (m/update-existing :parameter_mappings #(some->> % (lib/normalize ::parameters.schema/parameter-mappings)))(m/update-existing :visualization_settings mi/normalize-visualization-settings))))(defn virtual-card-settings@@ -180,14 +184,14 @@(dashboards.db/insert-dashcard-series! card-series))))(def ^:private DashboardCardUpdates- [:map- [:id ms/PositiveInt]- [:action_id {:optional true} [:maybe ms/PositiveInt]]- [:parameter_mappings {:optional true} [:maybe [:sequential :map]]]- [:visualization_settings {:optional true} [:maybe :map]]- [:inline_parameters {:optional true} [:maybe [:sequential ms/NonBlankString]]]- ;; series is a sequence of IDs of additional cards after the first to include as "additional serieses"- [:series {:optional true} [:maybe [:sequential ms/PositiveInt]]]])+ [:merge+ ::dashboards.schema/dashboard-card.update+ [:map {:closed true}+ [:id ms/PositiveInt]+ ;; series is a sequence of IDs of additional cards after the first to include as "additional serieses"+ [:series {:optional true} [:maybe [:sequential ms/PositiveInt]]]+ [:card {:optional true} [:maybe ::queries.schema/card]]+ [:collection_authority_level {:optional true} [:maybe [:or :keyword :string]]]]])(defn- shallow-updates"Returns the keys in `new` that have different values than the corresponding keys in `old`"@@ -216,15 +220,14 @@nil)))(def ^:private NewDashboardCard- ;; TODO - make the rest of the options explicit instead of just allowing whatever for other keys (#40021)- [:map- [:dashboard_id ms/PositiveInt]- [:action_id {:optional true} [:maybe ms/PositiveInt]]- ;; TODO - use ParamMapping. Breaks too many tests right now tho (#40021)- [:parameter_mappings {:optional true} [:maybe [:sequential map?]]]- [:visualization_settings {:optional true} [:maybe map?]]- [:inline_parameters {:optional true} [:maybe [:sequential ms/NonBlankString]]]- [:series {:optional true} [:maybe [:sequential ms/PositiveInt]]]])+ [:merge+ ::dashboards.schema/dashboard-card.update+ [:map {:closed true}+ [:dashboard_id ms/PositiveInt]+ [:id {:optional true} [:maybe ms/Int]]+ [:card {:optional true} [:maybe [:ref ::queries.schema/card]]]+ [:collection_authority_level {:optional true} [:maybe [:or :keyword :string]]]+ [:series {:optional true} [:maybe [:sequential ms/PositiveInt]]]]])(mu/defn create-dashboard-cards!"Create a new DashboardCard by inserting it into the database along with all associated pieces of data such asdiff --git a/src/metabase/dashboards/models/dashboard_card_series.clj b/src/metabase/dashboards/models/dashboard_card_series.cljindex a950b215d1e..a2baf27d580 100644--- a/src/metabase/dashboards/models/dashboard_card_series.clj+++ b/src/metabase/dashboards/models/dashboard_card_series.clj@@ -1,5 +1,6 @@(ns metabase.dashboards.models.dashboard-card-series(:require+ [metabase.dashboards.schema][metabase.models.serialization :as serdes][methodical.core :as methodical][toucan2.core :as t2]))diff --git a/src/metabase/dashboards/models/dashboard_tab.clj b/src/metabase/dashboards/models/dashboard_tab.cljindex 4275209074a..abefe6676ef 100644--- a/src/metabase/dashboards/models/dashboard_tab.clj+++ b/src/metabase/dashboards/models/dashboard_tab.clj@@ -3,6 +3,7 @@[medley.core :as m][metabase.dashboards.db :as dashboards.db][metabase.dashboards.models.dashboard-card :as dashboard-card]+ [metabase.dashboards.schema :as dashboards.schema][metabase.models.interface :as mi][metabase.models.serialization :as serdes][metabase.util :as u]@@ -64,7 +65,7 @@(mu/defn create-tabs! :- [:map-of neg-int? pos-int?]"Create the new tabs and returned a mapping from temporary tab ID to the new tab ID."[dashboard-id :- ms/PositiveInt- new-tabs :- [:sequential [:map [:id neg-int?]]]]+ new-tabs :- [:sequential [:merge ::dashboards.schema/dashboard-tab.update [:map {:closed true} [:id neg-int?]]]]](let [new-tab-ids (dashboards.db/insert-dashboard-tabs! (->> new-tabs(map #(dissoc % :id))(map #(assoc % :dashboard_id dashboard-id))))]@@ -72,8 +73,8 @@(mu/defn update-tabs! :- nil?"Updates tabs of a dashboard if changed."- [current-tabs :- [:sequential [:map [:id ms/PositiveInt]]]- new-tabs :- [:sequential [:map [:id ms/PositiveInt]]]]+ [current-tabs :- [:sequential ::dashboards.schema/dashboard-tab]+ new-tabs :- [:sequential ::dashboards.schema/dashboard-tab]](let [update-ks [:name :position]id->current-tab (m/index-by :id current-tabs)to-update-tabs (filterdiff --git a/src/metabase/dashboards/schema.clj b/src/metabase/dashboards/schema.cljindex 5db2505585b..26addd126a9 100644--- a/src/metabase/dashboards/schema.clj+++ b/src/metabase/dashboards/schema.clj@@ -1,5 +1,8 @@(ns metabase.dashboards.schema(:require+ [metabase.actions.schema]+ [metabase.collections.schema]+ [metabase.content-verification.schema][metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.parameter :as lib.schema.parameter][metabase.parameters.schema :as parameters.schema]@@ -7,11 +10,6 @@[metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms]))-(mr/def ::dashcard- [:map- [:id {:optional true} ::lib.schema.id/dashcard]- [:card {:optional true} [:ref ::queries.schema/card]]])-(mr/def ::parameters[:sequential;; the same as the normal parameters schema, but type is optional here.@@ -20,15 +18,39 @@[:map[:type {:optional true} [:ref ::lib.schema.parameter/type]]]]])-(mr/def ::dashboard- [:map- [:id {:optional true} ::lib.schema.id/dashboard]- [:parameters {:optional true} [:maybe ::parameters]]- [:dashcards {:optional true} [:maybe [:sequential ::dashcard]]]])-(mr/def ::dashboard.parameter"One entry of the `:parameters` column of a Dashboard, decoded."- :map)+ ::parameters.schema/parameter)++(mr/def ::dashboard+ "A Dashboard as selected from the app DB: every column of `:report_dashboard` (see `::dashboard.update`) plus+ `:id` and the keys some callers hydrate onto it."+ [:merge+ ::dashboard.update+ [:map {:closed true}+ [:id ::lib.schema.id/dashboard]+ [:moderation_status {:optional true} [:maybe [:or :keyword :string]]]+ [:resolved-params {:optional true} [:maybe [:map-of ms/NonBlankString ::parameters.schema/resolved-parameter]]]+ [:dashcards {:optional true} [:maybe [:sequential [:ref ::dashboard-card]]]]+ [:tabs {:optional true} [:maybe [:sequential ::dashboard-tab]]]+ [:collection_authority_level {:optional true} [:maybe [:or :keyword :string]]]+ [:can_write {:optional true} :boolean]+ [:can_restore {:optional true} :boolean]+ [:can_delete {:optional true} :boolean]+ [:can_set_cache_policy {:optional true} :boolean]+ [:param_fields {:optional true} [:maybe [:map-of :string [:sequential ::queries.schema/param-field]]]]+ [:is_remote_synced {:optional true} :boolean]+ [:moderation_reviews {:optional true} [:sequential :metabase.content-verification.schema/moderation-review]]+ [:collection {:optional true} [:maybe :metabase.collections.schema/collection-or-root]]+ [:last_used_param_values {:optional true} [:maybe [:map-of :string [:maybe :metabase.users.schema/user-parameter-value.value]]]]+ [:creator {:optional true} [:maybe :metabase.users.schema/user]]+ [:last-edit-info {:optional true} [:maybe+ [:map {:closed true}+ [:timestamp [:maybe ms/TemporalInstant]]+ [:id [:maybe ms/PositiveInt]]+ [:first_name [:maybe :string]]+ [:last_name [:maybe :string]]+ [:email [:maybe :string]]]]]]])(mr/def ::dashboard.update"What an update (or insert) of a Dashboard accepts: every column of `:report_dashboard` except `id`, all optional."@@ -63,30 +85,26 @@(mr/def ::dashboard-card.parameter-mapping"One entry of the `:parameter_mappings` column of a DashboardCard, decoded."- :map)+ ::parameters.schema/parameter-mapping)(mr/def ::dashboard-card.visualization-settings"The `:visualization_settings` column of a DashboardCard, decoded."- :map)+ ms/VisualizationSettings)(mr/def ::dashboard-card- "A DashboardCard as selected from the app DB: every column of `:report_dashboardcard`."- [:map {:closed true}- [:id ::lib.schema.id/dashcard]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]- [:size_x :int]- [:size_y :int]- [:row :int]- [:col :int]- [:card_id [:maybe ::lib.schema.id/card]]- [:dashboard_id ::lib.schema.id/dashboard]- [:parameter_mappings [:sequential ::dashboard-card.parameter-mapping]]- [:visualization_settings ::dashboard-card.visualization-settings]- [:entity_id :string]- [:action_id [:maybe ::lib.schema.id/action]]- [:dashboard_tab_id [:maybe ms/PositiveInt]]- [:inline_parameters [:maybe [:sequential :string]]]])+ "A DashboardCard as selected from the app DB: every column of `:report_dashboardcard`, plus the keys some callers+ hydrate onto it."+ [:merge+ ::dashboard-card.update+ [:map {:closed true}+ [:id ::lib.schema.id/dashcard]+ [:collection_authority_level {:optional true} [:maybe [:or :keyword :string]]]+ [:card {:optional true} [:maybe [:ref ::queries.schema/card]]]+ [:series {:optional true} [:maybe [:sequential [:ref ::queries.schema/card]]]]+ [:action {:optional true} [:maybe [:merge+ :metabase.actions.schema/action+ [:map {:closed true}+ [:database_enabled_actions {:optional true} :boolean]]]]]]])(mr/def ::dashboard-card.update"What an update (or insert) of a DashboardCard accepts: every column of `:report_dashboardcard` except `id`, all optional."@@ -108,11 +126,10 @@(mr/def ::dashboard-card-series"A DashboardCardSeries as selected from the app DB: every column of `:dashboardcard_series`."- [:map {:closed true}- [:id ms/PositiveInt]- [:dashboardcard_id ::lib.schema.id/dashcard]- [:card_id ::lib.schema.id/card]- [:position :int]])+ [:merge+ ::dashboard-card-series.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::dashboard-card-series.update"What an update (or insert) of a DashboardCardSeries accepts: every column of `:dashboardcard_series` except `id`, all optional."@@ -123,14 +140,10 @@(mr/def ::dashboard-tab"A DashboardTab as selected from the app DB: every column of `:dashboard_tab`."- [:map {:closed true}- [:id ms/PositiveInt]- [:dashboard_id ::lib.schema.id/dashboard]- [:name :string]- [:position :int]- [:entity_id :string]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]])+ [:merge+ ::dashboard-tab.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::dashboard-tab.update"What an update (or insert) of a DashboardTab accepts: every column of `:dashboard_tab` except `id`, all optional."diff --git a/src/metabase/dashboards/write.clj b/src/metabase/dashboards/write.cljindex 1ecbf09c4a6..53e2e32a204 100644--- a/src/metabase/dashboards/write.clj+++ b/src/metabase/dashboards/write.clj@@ -16,9 +16,11 @@[metabase.dashboards.models.dashboard-tab :as dashboard-tab][metabase.embedding.validation :as embedding.validation][metabase.events.core :as events]+ [metabase.lib-be.schema :as lib-be.schema]+ [metabase.lib.core :as lib]+ [metabase.lib.schema.common :as lib.schema.common][metabase.lib.schema.id :as lib.schema.id][metabase.models.interface :as mi]- [metabase.parameters.core :as parameters][metabase.parameters.params :as params][metabase.parameters.schema :as parameters.schema][metabase.pulse.broken-subscriptions :as pulse.broken-subscriptions]@@ -71,7 +73,7 @@Requires create permission on `:collection_id` (nil = root). Publishes `:event/dashboard-create`."[{:keys [name description parameters cache_ttl collection_id collection_positionwidth auto_apply_filters]}- :- [:map+ :- [:map {:closed true}[:name ms/NonBlankString][:parameters {:optional true} [:maybe ::parameters.schema/parameters]][:description {:optional true} [:maybe :string]]@@ -96,7 +98,7 @@dash (t2/with-transaction [_conn];; Adding a new dashboard at `collection_position` could cause other dashboards in this;; collection to change position, check that and fix up if needed- (api/maybe-reconcile-collection-position! dashboard-data)+ (api/maybe-reconcile-collection-position! (select-keys dashboard-data [:collection_id :collection_position]));; Ok, now save the Dashboard(dashboards.db/insert-dashboard! dashboard-data))](events/publish-event! :event/dashboard-create {:object dash :user-id api/*current-user-id*})@@ -109,14 +111,15 @@"You must be a superuser to change the value of `enable_embedding`, `embedding_type` or `embedding_params`. Embedding must beenabled."[dash-before-update dash-updates]- (when (or (api/column-will-change? :enable_embedding dash-before-update dash-updates)- (api/column-will-change? :embedding_type dash-before-update dash-updates)- (api/column-will-change? :embedding_params dash-before-update dash-updates))+ (when (or (api/column-will-change? (:enable_embedding dash-before-update) (get dash-updates :enable_embedding ::api/not-provided))+ (api/column-will-change? (:embedding_type dash-before-update) (get dash-updates :embedding_type ::api/not-provided))+ (api/column-will-change? (:embedding_params dash-before-update) (get dash-updates :embedding_params ::api/not-provided)))(embedding.validation/check-embedding-enabled)(api/check-superuser)))(mu/defn- param-target->field-id :- [:maybe ::lib.schema.id/field]- [target query]+ [target :- ::lib.schema.common/possibly-unnormalized-clause+ query :- [:maybe ::lib-be.schema/maybe-legacy-or-empty-query]](params/param-target->field-id target {:dataset_query query}));; TODO -- should we only check *new* or *modified* mappings?@@ -165,7 +168,7 @@[dashboard-id dashcards](let [dashcard-id->existing-mappings (existing-parameter-mappings dashboard-id)existing-mapping? (fn [dashcard-id mapping]- (let [mapping (parameters/normalize-parameter-mapping mapping)+ (let [mapping (lib/normalize ::parameters.schema/parameter-mapping mapping)existing-mappings (get dashcard-id->existing-mappings dashcard-id)](contains? existing-mappings (select-keys mapping [:target :parameter_id]))))new-mappings (for [{mappings :parameter_mappings, dashcard-id :id} dashcards@@ -316,7 +319,7 @@(t2/with-transaction [_conn];; If the dashboard has an updated position, or if the dashboard is moving to a new collection, we might need to;; adjust the collection position of other dashboards in the collection- (api/maybe-reconcile-collection-position! current-dash dash-updates)+ (api/maybe-reconcile-collection-position! (select-keys current-dash [:collection_id :collection_position]) (select-keys dash-updates [:collection_id :collection_position]))(when-let [updates (not-empty(u/select-keys-whendash-updates@@ -415,7 +418,7 @@Questions (questions stored 'in' the dashboard rather than a collection) and reference the rest (assumingpermissions)."[deep-copy? :- ms/MaybeBooleanValue- dashcards :- [:sequential :any]]+ dashcards :- [:sequential :metabase.dashboards.schema/dashboard-card]](let [card->cards (fn [{:keys [card series]}] (into [card] series))readable? (fn [card] (and (mi/model card) (mi/can-read? card)))card->decision (fn [parent-card card]@@ -588,7 +591,7 @@dashboard (t2/with-transaction [_conn];; Adding a new dashboard at `collection_position` could cause other dashboards in this;; collection to change position, check that and fix up if needed- (api/maybe-reconcile-collection-position! dashboard-data)+ (api/maybe-reconcile-collection-position! (select-keys dashboard-data [:collection_id :collection_position]));; Ok, now save the Dashboard(let [dash (dashboards.db/insert-dashboard! dashboard-data){id->new-card :copied@@ -617,5 +620,5 @@(when-let [newly-created-cards (seq @new-cards)](doseq [card newly-created-cards](events/publish-event! :event/card-create {:object card :user-id api/*current-user-id*})))- (events/publish-event! :event/dashboard-create {:object dashboard :user-id api/*current-user-id*})+ (events/publish-event! :event/dashboard-create {:object (dissoc dashboard :uncopied) :user-id api/*current-user-id*})dashboard))diff --git a/src/metabase/dashboards_rest/api.clj b/src/metabase/dashboards_rest/api.cljindex 758faeafb59..2c928793e4a 100644--- a/src/metabase/dashboards_rest/api.clj+++ b/src/metabase/dashboards_rest/api.clj@@ -26,11 +26,11 @@[metabase.events.core :as events][metabase.lib-be.core :as lib-be][metabase.lib-be.schema :as lib-be.schema]+ [metabase.lib.core :as lib][metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.parameter :as lib.schema.parameter][metabase.models.interface :as mi][metabase.parameters.chain-filter :as chain-filter]- [metabase.parameters.core :as parameters][metabase.parameters.dashboard :as parameters.dashboard][metabase.parameters.params :as params][metabase.parameters.schema :as parameters.schema]@@ -38,6 +38,7 @@[metabase.public-sharing.validation :as public-sharing.validation][metabase.pulse.core :as pulse][metabase.queries.core :as queries]+ [metabase.queries.schema :as queries.schema][metabase.query-permissions.core :as query-perms][metabase.query-processor.api :as api.dataset][metabase.query-processor.dashboard :as qp.dashboard]@@ -189,7 +190,7 @@dash (t2/with-transaction [_conn];; Adding a new dashboard at `collection_position` could cause other dashboards in this;; collection to change position, check that and fix up if needed- (api/maybe-reconcile-collection-position! dashboard-data)+ (api/maybe-reconcile-collection-position! (select-keys dashboard-data [:collection_id :collection_position]));; Ok, now save the Dashboard(dashboards-rest.db/insert-dashboard! dashboard-data))](events/publish-event! :event/dashboard-create {:object dash :user-id api/*current-user-id*})@@ -425,7 +426,7 @@Questions (questions stored 'in' the dashboard rather than a collection) and reference the rest (assumingpermissions)."[deep-copy? :- ms/MaybeBooleanValue- dashcards :- [:sequential :any]]+ dashcards :- [:sequential :metabase.dashboards.schema/dashboard-card]](let [card->cards (fn [{:keys [card series]}] (into [card] series))readable? (fn [card] (and (mi/model card) (mi/can-read? card)))card->decision (fn [parent-card card]@@ -593,7 +594,7 @@dashboard (t2/with-transaction [_conn];; Adding a new dashboard at `collection_position` could cause other dashboards in this;; collection to change position, check that and fix up if needed- (api/maybe-reconcile-collection-position! dashboard-data)+ (api/maybe-reconcile-collection-position! (select-keys dashboard-data [:collection_id :collection_position]));; Ok, now save the Dashboard(let [dash (dashboards-rest.db/insert-dashboard! dashboard-data){id->new-card :copied@@ -622,7 +623,7 @@(when-let [newly-created-cards (seq @new-cards)](doseq [card newly-created-cards](events/publish-event! :event/card-create {:object card :user-id api/*current-user-id*})))- (events/publish-event! :event/dashboard-create {:object dashboard :user-id api/*current-user-id*})+ (events/publish-event! :event/dashboard-create {:object (dissoc dashboard :uncopied) :user-id api/*current-user-id*})dashboard));;; --------------------------------------------- List public and embeddable dashboards ------------------------------@@ -728,9 +729,9 @@"You must be a superuser to change the value of `enable_embedding`, `embedding_type` or `embedding_params`. Embedding must beenabled."[dash-before-update dash-updates]- (when (or (api/column-will-change? :enable_embedding dash-before-update dash-updates)- (api/column-will-change? :embedding_type dash-before-update dash-updates)- (api/column-will-change? :embedding_params dash-before-update dash-updates))+ (when (or (api/column-will-change? (:enable_embedding dash-before-update) (get dash-updates :enable_embedding ::api/not-provided))+ (api/column-will-change? (:embedding_type dash-before-update) (get dash-updates :embedding_type ::api/not-provided))+ (api/column-will-change? (:embedding_params dash-before-update) (get dash-updates :embedding_params ::api/not-provided)))(embedding.validation/check-embedding-enabled)(api/check-superuser)))@@ -750,7 +751,8 @@api/generic-204-no-content)(mu/defn- param-target->field-id :- [:maybe ::lib.schema.id/field]- [target query]+ [target :- ::lib.schema.parameter/target+ query :- [:maybe ::queries.schema/card.dataset-query]](params/param-target->field-id target {:dataset_query query}));; TODO -- should we only check *new* or *modified* mappings?@@ -796,7 +798,7 @@[dashboard-id dashcards](let [dashcard-id->existing-mappings (existing-parameter-mappings dashboard-id)existing-mapping? (fn [dashcard-id mapping]- (let [mapping (parameters/normalize-parameter-mapping mapping)+ (let [mapping (lib/normalize ::parameters.schema/parameter-mapping mapping)existing-mappings (get dashcard-id->existing-mappings dashcard-id)](contains? existing-mappings (select-keys mapping [:target :parameter_id]))))new-mappings (for [{mappings :parameter_mappings, dashcard-id :id} dashcards@@ -1017,7 +1019,7 @@(t2/with-transaction [_conn];; If the dashboard has an updated position, or if the dashboard is moving to a new collection, we might need to;; adjust the collection position of other dashboards in the collection- (api/maybe-reconcile-collection-position! current-dash dash-updates)+ (api/maybe-reconcile-collection-position! (select-keys current-dash [:collection_id :collection_position]) (select-keys dash-updates [:collection_id :collection_position]))(when-let [updates (not-empty(u/select-keys-whendash-updates@@ -1449,7 +1451,7 @@(with-dashboard-load-id dashboard_load_id(m/mapply qp.dashboard/process-query-for-dashcard(merge- body+ (dissoc body :dashboard_load_id){:dashboard (api/check-404 (dashboards-rest.db/dashboard dashboard-id)):card (api/check-404 (dashboards-rest.db/card card-id)):dashcard (api/check-404 (dashboards-rest.db/dashcard dashcard-id))}))))diff --git a/src/metabase/documents/api/document.clj b/src/metabase/documents/api/document.cljindex be8b7b8c77a..49a2a436cd9 100644--- a/src/metabase/documents/api/document.clj+++ b/src/metabase/documents/api/document.clj@@ -129,7 +129,7 @@(when-not (contains? body :archived)(api/check-not-archived existing-document))(api/write-check existing-document)- (when (api/column-will-change? :collection_id existing-document body)+ (when (api/column-will-change? (:collection_id existing-document) (get body :collection_id ::api/not-provided))(m.document/validate-collection-move-permissions (:collection_id existing-document) collection_id))(m.document/update-document! existing-document body)))@@ -206,7 +206,7 @@:collection_position collection_position}new-document (t2/with-transaction [_conn](when collection_position- (api/maybe-reconcile-collection-position! document-data))+ (api/maybe-reconcile-collection-position! (select-keys document-data [:collection_id :collection_position])))(let [new-document-id (documents.db/insert-document! document-data)card-id-map (copy-cards-for-document! from-document-id new-document-id collection_id)](when (seq card-id-map)diff --git a/src/metabase/documents/db.clj b/src/metabase/documents/db.cljindex 4770d332f1b..c27e61f1bd2 100644--- a/src/metabase/documents/db.clj+++ b/src/metabase/documents/db.clj@@ -141,7 +141,7 @@(mu/defn table"The Table with `id`, or nil."- [id :- ::lib.schema.id/table]+ [id :- [:maybe ::lib.schema.id/table]](t2/select-one :model/Table :id id {:from [(warehouse-schema-overlay/table-query)]}))(mu/defn dashboarddiff --git a/src/metabase/documents/models/document.clj b/src/metabase/documents/models/document.cljindex 9e779be100b..70ad861cd81 100644--- a/src/metabase/documents/models/document.clj+++ b/src/metabase/documents/models/document.clj@@ -6,6 +6,7 @@[metabase.collections.models.collection :as collection][metabase.documents.db :as documents.db][metabase.documents.prose-mirror :as prose-mirror]+ [metabase.documents.schema :as documents.schema][metabase.events.core :as events][metabase.lib-be.schema :as lib-be.schema][metabase.models.interface :as mi]@@ -204,8 +205,8 @@[:content_type :string]]"Rewrite the card FK (`[:attrs \"id\"]`) of every cardEmbed node found in `card-id-map`.Touches nothing else on the node — in particular a node's `:_id` never changes here."- [document :- [:map- [:document :any]+ [document :- [:map {:closed true}+ [:document ::documents.schema/document.document][:content_type :string]]card-id-map :- [:maybe [:map-of :int ms/PositiveInt]]](cond-> document@@ -231,7 +232,7 @@[cards-to-create :- [:map-of [:int {:max -1}] CardCreateSchema]document-id :- ms/PositiveIntdocument-collection-id :- [:or :nil ms/PositiveInt]- creator :- [:map [:id ms/PositiveInt]]]+ creator :- :metabase.users.schema/user](when (seq cards-to-create)(reduce-kv(fn [result-map original-key card-data]@@ -255,7 +256,7 @@Returns:- map of old-card-id -> cloned-card-id"- [{:keys [id collection_id] :as document}]+ [{:keys [id collection_id] :as document} :- ::documents.schema/document](let [card-ids (prose-mirror/collect-ast document #(when (and (= prose-mirror/card-embed-type (:type %))(pos-int? (get (:attrs %) "id")))(get (:attrs %) "id")))@@ -315,9 +316,9 @@(`api/create-check`) are the caller's job, run before this — the same split the REST`POST /api/document/` handler uses."[{:keys [name document collection_id collection_position cards]}- :- [:map+ :- [:map {:closed true}[:name DocumentName]- [:document :any]+ [:document ::documents.schema/document.document][:collection_id {:optional true} [:maybe ms/PositiveInt]][:collection_position {:optional true} [:maybe ms/PositiveInt]][:cards {:optional true} [:maybe [:map-of [:int {:max -1}] CardCreateSchema]]]]]@@ -370,11 +371,11 @@and return the updated document. Permission checks (write-check, archived state,collection-move) are the caller's job, run before this — the same split the REST`PUT /api/document/:id` handler uses."- [existing-document :- [:map [:id ms/PositiveInt]]+ [existing-document :- ::documents.schema/document{:keys [name document collection_id collection_position cards] :as body}- :- [:map+ :- [:map {:closed true}[:name {:optional true} DocumentName]- [:document {:optional true} :any]+ [:document {:optional true} ::documents.schema/document.document][:collection_id {:optional true} [:maybe ms/PositiveInt]][:collection_position {:optional true} [:maybe ms/PositiveInt]];; Any int key, matching the REST `DocumentUpdateOptions` this backs: with no `:document`@@ -386,10 +387,10 @@document-updates (dissoc (api/updates-with-archived-directly existing-document body) :cards)](t2/with-transaction [_conn](when collection_position- (api/maybe-reconcile-collection-position! existing-document {:collection_id (if (contains? body :collection_id)- collection_id- (:collection_id existing-document))- :collection_position collection_position}))+ (api/maybe-reconcile-collection-position! (select-keys existing-document [:collection_id :collection_position]) {:collection_id (if (contains? body :collection_id)+ collection_id+ (:collection_id existing-document))+ :collection_position collection_position}))(let [card-id-map (when document(merge(clone-cards-in-document! (assoc existing-document :document document))@@ -643,17 +644,18 @@;; NOTE: unlike the readers below, this feeds `deserialization-dependencies`, which runs on the already-serialized;; form where `:entityId` is a serdes path (a vector of {:model :id} maps), not a raw id — so it is not guarded;; with `node-entity-id` here.- (set (prose-mirror/collect-ast document (fn document-deps [{:keys [type attrs]}]- (cond- (and (= prose-mirror/smart-link-type type)- (contains? model->serdes-model (get attrs "model")))- (get attrs "entityId")-- (= prose-mirror/card-embed-type type)- (get attrs "id")-- :else- nil))))))+ (set (prose-mirror/collect-ast (update document :document prose-mirror/normalize-document)+ (fn document-deps [{:keys [type attrs]}]+ (cond+ (and (= prose-mirror/smart-link-type type)+ (contains? model->serdes-model (get attrs "model")))+ (get attrs "entityId")++ (= prose-mirror/card-embed-type type)+ (get attrs "id")++ :else+ nil))))))(defmethod serdes/deserialization-dependencies "Document"[{:keys [collection_id] :as document}]diff --git a/src/metabase/documents/schema.clj b/src/metabase/documents/schema.cljindex 45826535fb7..e7185b0b917 100644--- a/src/metabase/documents/schema.clj+++ b/src/metabase/documents/schema.clj@@ -1,5 +1,7 @@(ns metabase.documents.schema(:require+ [metabase.collections.schema]+ [metabase.documents.prose-mirror :as prose-mirror][metabase.lib.schema.id :as lib.schema.id][metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms]))@@ -10,30 +12,21 @@(mr/def ::document.document"The `:document` column of a Document, decoded."- :map)+ ::prose-mirror/ast)(mr/def ::document- "A Document as selected from the app DB: every column of `:document`."- [:map {:closed true}- [:id ms/PositiveInt]- [:name :string]- [:created_at ms/TemporalInstant]- [:document [:maybe ::document.document]]- [:content_type [:or :keyword :string]]- [:creator_id ::lib.schema.id/user]- [:updated_at ms/TemporalInstant]- [:collection_id [:maybe ::lib.schema.id/collection]]- [:archived :boolean]- [:archived_directly [:maybe :boolean]]- [:entity_id :string]- [:last_viewed_at ms/TemporalInstant]- [:view_count :int]- [:collection_position [:maybe :int]]- [:public_uuid [:maybe :string]]- [:made_public_by_id [:maybe ms/PositiveInt]]- [:public_uuid_prefix [:maybe :string]]- [:exploration_id [:maybe ms/PositiveInt]]- [:is_placeholder :boolean]])+ "A Document as selected from the app DB: every column of `:document`, plus `:creator` and `:collection` some+ callers hydrate onto it."+ [:merge+ ::document.update+ [:map {:closed true}+ [:id ms/PositiveInt]+ [:creator {:optional true} [:maybe :metabase.users.schema/user]]+ [:collection {:optional true} [:maybe :metabase.collections.schema/collection-or-root]]+ [:can_write {:optional true} :boolean]+ [:can_delete {:optional true} :boolean]+ [:can_restore {:optional true} :boolean]+ [:is_remote_synced {:optional true} :boolean]]])(mr/def ::document.update"What an update (or insert) of a Document accepts: every column of `:document` except `id`, all optional."diff --git a/src/metabase/driver.clj b/src/metabase/driver.cljindex e45be4996b1..412808a47d4 100644--- a/src/metabase/driver.clj+++ b/src/metabase/driver.clj@@ -529,8 +529,8 @@{:closed true}[:schema-names {:optional true} [:maybe[:or- [:sequential :string]- [:set :string]]]]+ [:sequential [:maybe :string]]+ [:set [:maybe :string]]]]][:table-names {:optional true} [:maybe[:or[:sequential :string]diff --git a/src/metabase/driver/common.clj b/src/metabase/driver/common.cljindex 78c227d5766..814791940ec 100644--- a/src/metabase/driver/common.clj+++ b/src/metabase/driver/common.clj@@ -485,7 +485,7 @@If `:my-driver` returns [[driver/db-start-of-week]] as `:sunday` (1 is Sunday, 2 is Monday, and so forth),and [[metabase.lib-be.core/start-of-week]] is `:monday` (the results should have 1 as Monday, 2 as Tuesday... 7 isSunday), then the offset should be `-1`, because `:monday` returned by the driver (`2`) minus `1` = `1`."- [driver]+ [driver :- :keyword](start-of-week-offset-for-day (driver/db-start-of-week driver)))(defn json-unfolding-defaultdiff --git a/src/metabase/driver/common/table_rows_sample.clj b/src/metabase/driver/common/table_rows_sample.cljindex a4b2623adea..7ed1ec71ffe 100644--- a/src/metabase/driver/common/table_rows_sample.clj+++ b/src/metabase/driver/common/table_rows_sample.clj@@ -5,18 +5,14 @@[metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]+ [metabase.lib.ref :as lib.ref][metabase.lib.schema :as lib.schema]- [metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.metadata :as lib.schema.metadata]+ [metabase.lib.schema.order-by :as lib.schema.order-by][metabase.query-processor.core :as qp][metabase.query-processor.schema :as qp.schema][metabase.util.malli :as mu][metabase.util.malli.registry :as mr]- ;; TODO -- for historical reasons this stuff uses the Toucan models instead of the QP Metadata Store and- ;; `:metadata/*` models -- at some point we should fix this. [[driver/table-rows-sample]] is called by sync however- ;; so we need to go in and update the sync code as well.- ^{:clj-kondo/ignore [:discouraged-namespace]}- [metabase.util.malli.schema :as ms][metabase.warehouse-schema.metadata-queries :as schema.metadata-queries]))(def max-sample-rows@@ -31,10 +27,10 @@(mr/def ::table-rows-sample.options"Schema for `table-rows-sample` options"[:maybe- [:map+ [:map {:closed true}[:truncation-size {:optional true} :int][:limit {:optional true} :int]- [:order-by {:optional true} [:maybe [:sequential :some]]] ; something that can be passed to [[metabase.lib.core/order-by]]+ [:order-by {:optional true} [:maybe [:sequential [:or ::lib.schema.order-by/order-by ::lib.ref/referenceable]]]][:rff {:optional true} fn?]]])(mu/defn- table-rows-sample-query :- ::lib.schema/query@@ -87,17 +83,13 @@`:truncation-size`: [optional] size to truncate text fields if the driver supports expressions.`:rff`: [optional] a reducing function function (a function that given initial results metadata returns a reducingfunction) to reduce over the result set in the the query-processor rather than realizing the whole collection"- ([table :- (ms/InstanceOf :model/Table)- fields :- [:sequential (ms/InstanceOf :model/Field)]- rff]+ ([table :- :metabase.warehouse-schema.schema/table+ fields :- [:sequential :metabase.warehouse-schema.schema/field]+ rff :- ::qp.schema/rff](table-rows-sample table fields rff nil))- ([table :- [:and- (ms/InstanceOf :model/Table)- [:map- [:id ::lib.schema.id/table]- [:db_id ::lib.schema.id/database]]]- fields :- [:sequential (ms/InstanceOf :model/Field)]+ ([table :- :metabase.warehouse-schema.schema/table+ fields :- [:sequential :metabase.warehouse-schema.schema/field]rff :- ::qp.schema/rffopts :- ::table-rows-sample.options](let [database-id (:db_id table)diff --git a/src/metabase/driver/ddl/interface.clj b/src/metabase/driver/ddl/interface.cljindex db5aec3f305..6aa1c6c5091 100644--- a/src/metabase/driver/ddl/interface.clj+++ b/src/metabase/driver/ddl/interface.clj@@ -4,10 +4,10 @@[metabase.appearance.core :as appearance][metabase.driver :as driver][metabase.lib.schema.common :as lib.schema.common]- [metabase.lib.schema.id :as lib.schema.id][metabase.system.core :as system][metabase.util.i18n :refer [tru]]- [metabase.util.malli :as mu])+ [metabase.util.malli :as mu]+ [metabase.warehouses.schema :as warehouses.schema])(:import(java.time Instant)(java.time.format DateTimeFormatter)))@@ -19,7 +19,7 @@multiple connections from multiple metabases remain distinct. The UUID will have the first character of each section taken.(schema-name {:id 234} \"143dd8ce-e116-4c7f-8d6d-32e99eaefbbc\") -> \"metabase_cache_1e483_1\""- [{:keys [id] :as _database} :- [:map [:id ::lib.schema.id/database]]+ [{:keys [id] :as _database} :- ::warehouses.schema/database-or-metadatasite-uuid-string :- ::lib.schema.common/non-blank-string](let [instance-string (apply str (map first (str/split site-uuid-string #"-")))](format "metabase_cache_%s_%s" instance-string id)))diff --git a/src/metabase/driver/h2.clj b/src/metabase/driver/h2.cljindex d18d81385c5..825292bdcf6 100644--- a/src/metabase/driver/h2.clj+++ b/src/metabase/driver/h2.clj@@ -190,8 +190,7 @@(let [[_ {:strs [USER]}] (connection-string->file+options db)]USER)))-(mu/defn- check-native-query-not-using-default-user [{query-type :type, :as query} :- [:map- [:type [:enum :native :query]]]]+(mu/defn- check-native-query-not-using-default-user [{query-type :type, :as query} :- :metabase.lib.util/legacy-query](u/prog1 query;; For :native queries check to make sure the DB in question has a (non-default) NAME property specified in the;; connection string. We don't allow SQL execution on H2 databases for the default admin account for security@@ -341,11 +340,7 @@CommandInterface/CALL} cmd-type-nums)(nil? remaining-sql)))))-(mu/defn- check-read-only-statements [{{sql :query} :native, :as _query} :- [:map- [:type [:enum :query :native]]- [:native- [:map- [:query string?]]]]]+(mu/defn- check-read-only-statements [{{sql :query} :native, :as _query} :- :metabase.lib.util/legacy-query](when sql(check-no-unsupported-functions sql)(let [query-classification (classify-query (driver-api/database (driver-api/metadata-provider))@@ -362,10 +357,7 @@(mu/defmethod driver/execute-write-query! :h2[driver :- :keyword- query :- [:map- [:type [:= :native]]- [:native [:map- [:query :string]]]]]+ query :- :metabase.lib.util/legacy-query](check-native-query-not-using-default-user query)(check-action-commands-allowed query)((get-method driver/execute-write-query! :sql-jdbc) driver query))diff --git a/src/metabase/driver/mysql.clj b/src/metabase/driver/mysql.cljindex 5aef1ed1a3b..243877299fc 100644--- a/src/metabase/driver/mysql.clj+++ b/src/metabase/driver/mysql.clj@@ -36,6 +36,7 @@[metabase.util.malli :as mu][metabase.util.memoize :as memoize][metabase.util.performance :as perf :refer [get-in mapv not-empty some]]+ [metabase.warehouses.schema :as warehouses.schema][next.jdbc :as next.jdbc])(:import(java.io File)@@ -135,7 +136,7 @@(= driver :mysql)))(mu/defn- database-flavor :- [:maybe :string]- ^String [database :- [:maybe :map]]+ ^String [database :- [:maybe [:or driver-api/schema.metadata.database ::warehouses.schema/database ::warehouses.schema/database.update]]];; avoid trying `:dbms_version` if `:dbms-version` is present but `nil`; this will cause snake-hating-map warnings(when-let [k (some #(when (contains? database %)%)diff --git a/src/metabase/driver/postgres.clj b/src/metabase/driver/postgres.cljindex 3aafc6c9a34..d886bb91453 100644--- a/src/metabase/driver/postgres.clj+++ b/src/metabase/driver/postgres.clj@@ -537,7 +537,7 @@(mu/defn- date-trunc[unit :- driver-api/schema.temporal-bucketing.unit.date-time.truncate- expr]+ expr :- ::h2x/expr];; Branches are ordered most-specific-first because `database-or-effective-type-isa?` checks `isa?` on the effective;; type fallback: `:type/TimeWithTZ` is a descendant of `:type/Time`, so the timetz branch must run first to avoid a;; nested-source-query `timetz` column being routed to the plain-time path (#75193, #68065).@@ -609,7 +609,7 @@(mu/defn- enum-cast[database-type :- driver-api/schema.common.non-blank-string- raw-value]+ raw-value :- ::h2x/expr](-> [:cast raw-value (apply h2x/identifier :type-name (enum-type-components database-type))](h2x/with-database-type-info database-type)))diff --git a/src/metabase/driver/sql/parameters/substitute.clj b/src/metabase/driver/sql/parameters/substitute.cljindex 916807bd1da..27e785f5716 100644--- a/src/metabase/driver/sql/parameters/substitute.clj+++ b/src/metabase/driver/sql/parameters/substitute.clj@@ -6,6 +6,7 @@[metabase.driver-api.core :as driver-api][metabase.driver.sql.parameters.substitution :as sql.params.substitution][metabase.lib.core :as lib]+ [metabase.lib.schema.literal :as lib.schema.literal][metabase.lib.schema.metadata :as lib.schema.metadata][metabase.query-processor.store :as qp.store][metabase.util.i18n :refer [tru]]@@ -26,15 +27,17 @@[:maybe[:tuple#_sql [:maybe string?]- #_args [:maybe [:sequential any?]]- #_missing [:maybe any?]]])+ #_args [:maybe [:sequential ::lib.schema.literal/param-value]]+ #_missing [:maybe [:sequential :string]]]])(mu/defn- substitute-field-param :- ::acc- [metadata-providerable- [sql args missing] :- ::acc- in-optional?- k- {:keys [_field value], :as v}]+ [metadata-providerable :- ::lib.schema.metadata/metadata-providerable+ [sql args missing] :- ::acc+ in-optional? :- :boolean+ k :- :string+ {:keys [_field value], :as v} :- [:or+ :metabase.lib.parameters.parse.types/field-filter+ :metabase.lib.parameters.parse.types/temporal-unit]](if (and (= value lib/parsed-param-no-value-placeholder) in-optional?);; no-value field filters inside optional clauses are ignored, and eventually emitted entirely[sql args (conj missing k)]@@ -44,18 +47,21 @@[(str sql replacement-snippet) (concat args prepared-statement-args) missing])))(mu/defn- substitute-simple-query :- ::acc- [metadata-providerable- [sql args missing] :- ::acc- v]+ [metadata-providerable :- ::lib.schema.metadata/metadata-providerable+ [sql args missing] :- ::acc+ v :- [:or+ :metabase.lib.parameters.parse.types/referenced-card-query+ :metabase.lib.parameters.parse.types/referenced-table-query]](let [{:keys [replacement-snippet prepared-statement-args]}(->replacement-snippet-info metadata-providerable v)][(str sql replacement-snippet) (concat args prepared-statement-args) missing]))(mu/defn- substitute-native-query-snippet :- ::acc[metadata-providerable :- ::lib.schema.metadata/metadata-providerable- param->value- [sql args missing]- in-optional? v]+ param->value :- [:maybe [:map-of :string :metabase.lib.parameters.parse.types/parsed-value]]+ [sql args missing] :- ::acc+ in-optional? :- :boolean+ v :- :metabase.lib.parameters.parse.types/referenced-query-snippet](let [{:keys [replacement-snippet]} (->replacement-snippet-info metadata-providerable v)[processed-snippet snippet-args snippet-missing] (substitute*metadata-providerable@@ -68,9 +74,9 @@(mu/defn- substitute-param :- ::acc[metadata-providerable :- ::lib.schema.metadata/metadata-providerable- param->value- [sql args missing] :- ::acc- in-optional?+ param->value :- [:maybe [:map-of :string :metabase.lib.parameters.parse.types/parsed-value]]+ [sql args missing] :- ::acc+ in-optional? :- :boolean{:keys [k]} :- :metabase.lib.parameters.parse.types/param](if-not (contains? param->value k)[sql args (conj missing k)]@@ -97,9 +103,9 @@(mu/defn- substitute-optional :- ::acc[metadata-providerable :- ::lib.schema.metadata/metadata-providerable- param->value- [sql args missing] :- ::acc- {subclauses :args}]+ param->value :- [:maybe [:map-of :string :metabase.lib.parameters.parse.types/parsed-value]]+ [sql args missing] :- ::acc+ {subclauses :args} :- :metabase.lib.parameters.parse.types/optional](let [[opt-sql opt-args opt-missing] (substitute* metadata-providerable param->value subclauses true)](if (seq opt-missing)[sql args missing]@@ -108,9 +114,9 @@(mu/defn- substitute* :- ::acc"Returns a sequence of `[replaced-sql-string jdbc-args missing-parameters]`."[metadata-providerable :- ::lib.schema.metadata/metadata-providerable- param->value :- [:maybe [:map-of string? any?]]+ param->value :- [:maybe [:map-of :string :metabase.lib.parameters.parse.types/parsed-value]]parsed :- [:sequential :metabase.lib.parameters.parse/parsed-token]- in-optional?]+ in-optional? :- :boolean](reduce(fn [[sql args missing] x](cond@@ -138,7 +144,7 @@;; -> [\"select * from foobars where bird_type = ?\" [\"Steller's Jay\"]]"[metadata-providerable :- ::lib.schema.metadata/metadata-providerableparsed-query :- [:sequential :metabase.lib.parameters.parse/parsed-token]- param->value :- [:maybe [:map-of string? any?]]]+ param->value :- [:maybe [:map-of :string :metabase.lib.parameters.parse.types/parsed-value]]](log/tracef "Substituting %d params in query" (count param->value))(let [[sql args missing] (try(substitute* metadata-providerable param->value parsed-query false)diff --git a/src/metabase/driver/sql/parameters/substitution.clj b/src/metabase/driver/sql/parameters/substitution.cljindex d8459cbf56a..f72931068c0 100644--- a/src/metabase/driver/sql/parameters/substitution.clj+++ b/src/metabase/driver/sql/parameters/substitution.clj@@ -46,7 +46,8 @@(mu/defn make-stmt-subs :- PreparedStatementSubstitution"Create a `PreparedStatementSubstitution` map for `sql-string` and the `param-seq`"- [sql-string param-seq]+ [sql-string :- :string+ param-seq :- [:maybe [:sequential :metabase.lib.schema.common/field-value]]]{:sql-string sql-string:param-values param-seq})@@ -114,10 +115,26 @@;;; ------------------------------------------- ->replacement-snippet-info -------------------------------------------+(mr/def ::substitutable-value+ "A value (or nested sequence of values) that [[->replacement-snippet-info]] and [[->prepared-substitution]] know how+ to turn into SQL: a raw scalar, a temporal literal, or one of the structured parsed date shapes."+ [:or+ :string+ number?+ :boolean+ :keyword+ uuid?+ nil?+ (driver-api/instance-of-class Temporal)+ [:sequential [:ref ::substitutable-value]]+ :metabase.lib.parameters.parse.types/date+ :metabase.lib.parameters.parse.types/date-range+ :metabase.lib.parameters.parse.types/date-time-range])+(mr/def ::param-snippet-info- [:map+ [:map {:closed true}[:replacement-snippet {:optional true} :string] ; allowed to be blank if this is an optional param- [:prepared-statement-args {:optional true} [:maybe [:sequential :any]]]])+ [:prepared-statement-args {:optional true} [:maybe [:sequential :metabase.lib.schema.common/field-value]]]]);; TODO (Cam 2026-05-21) Update this to take an explicit `metadata-providerable`(defmulti ->replacement-snippet-info@@ -169,7 +186,7 @@:prepared-statement-args (apply concat (map :prepared-statement-args values))}))(mu/defn- maybe-parse-temporal-literal :- (driver-api/instance-of-class java.time.temporal.Temporal)- [x]+ [x :- [:or :string (driver-api/instance-of-class Temporal)]](condp instance? xString (u.date/parse x)Temporal x@@ -237,24 +254,28 @@;; for relative dates convert the param to a `DateRange` record type and call `->replacement-snippet-info` on it(mu/defn- date-range-field-filter->replacement-snippet-info :- ::param-snippet-info- [driver value]+ [driver :- :keyword+ value :- driver-api/schema.common.non-blank-string](let [{:keys [start end]} (params.dates/date-string->range value)](->> (lib/parsed-date-range-param start end)(->replacement-snippet-info driver))))(mu/defn- field-filter->equals-clause-sql :- ::param-snippet-info- [driver value]+ [driver :- :keyword+ value :- ::substitutable-value](-> (->replacement-snippet-info driver value)(update :replacement-snippet (partial str "= "))))(mu/defn- field-filter-multiple-values->in-clause-sql :- ::param-snippet-info- [driver values]+ [driver :- :keyword+ values :- [:sequential ::substitutable-value]](-> (->replacement-snippet-info driver (vec values))(update :replacement-snippet (partial format "IN (%s)"))))(mu/defn- honeysql->replacement-snippet-info :- ::param-snippet-info"Convert `hsql-form` to a replacement snippet info map by passing it to HoneySQL's `format` function."- [driver hsql-form]+ [driver :- :keyword+ hsql-form :- ::h2x/expr](let [[snippet & args] (sql.qp/format-honeysql driver hsql-form)]{:replacement-snippet snippet:prepared-statement-args args}))@@ -263,7 +284,7 @@[driver :- :keywordcol :- driver-api/schema.metadata.columnparam-type :- driver-api/schema.parameter.type- value]+ value :- ::substitutable-value];; The [[metabase.query-processor.middleware.parameters/substitute-parameters]] QP middleware actually happens before;; the [[metabase.query-processor.middleware.resolve-fields/resolve-fields]] middleware that would normally fetch all;; the Fields we need in a single pass, so this is actually necessary here. I don't think switching the order of the@@ -282,7 +303,10 @@"Return an appropriate snippet to represent this `field` in SQL given its param type.For non-date Fields, this is just a quoted identifier; for dates, the SQL includes appropriately bucketing based onthe `param-type`."- [driver field param-type value]+ [driver :- :keyword+ field :- driver-api/schema.metadata.column+ param-type :- driver-api/schema.parameter.type+ value :- ::substitutable-value](->> (field->field-ref driver field param-type value)(->honeysql driver)(honeysql->replacement-snippet-info driver)@@ -301,7 +325,9 @@(mu/defn- field-filter->replacement-snippet-info :- ::param-snippet-info"Return `[replacement-snippet & prepared-statement-args]` appropriate for a field filter parameter."- [driver {{param-type :type, value :value, :as params} :value, field :field, :as field-filter}]+ [driver :- :keyword+ {{param-type :type, value :value, :as params} :value, field :field, :as field-filter}+ :- :metabase.lib.parameters.parse.types/field-filter](assert (:id field) (format "Why doesn't Field have an ID?\n%s" (u/pprint-to-str field)))(letfn [(prepend-field [x](update x :replacement-snippetdiff --git a/src/metabase/driver/sql/query_processor.clj b/src/metabase/driver/sql/query_processor.cljindex fb9cbafb795..1cf04ff77f2 100644--- a/src/metabase/driver/sql/query_processor.clj+++ b/src/metabase/driver/sql/query_processor.clj@@ -17,6 +17,7 @@[metabase.lib.metadata.protocols :as lib.metadata.protocols][metabase.lib.options :as lib.options][metabase.lib.util :as lib.util]+ [metabase.query-processor.schema :as qp.schema][metabase.query-processor.util.add-alias-info :as add][metabase.query-processor.util.persisted-cache :as qp.persisted][metabase.util :as u]@@ -25,6 +26,7 @@[metabase.util.i18n :refer [tru]][metabase.util.log :as log][metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr][metabase.util.match :as match][metabase.util.performance :as perf :refer [empty? every? get-in mapv not-empty select-keys some]][potemkin :as p]@@ -493,8 +495,8 @@(truncate-fn expr) => truncated-expr"[driver :- :keyword- truncate-fn :- [:=> [:cat :any] :any]- expr]+ truncate-fn :- [:=> [:cat ::h2x/expr] ::h2x/expr]+ expr :- ::h2x/expr](let [offset (driver.common/start-of-week-offset driver)](if (not= offset 0)(add-interval-honeysql-form driver@@ -513,16 +515,19 @@This assumes `day-of-week` as returned by the driver is already between `1` and `7` (adjust it if it's not). Itadjusts as needed to match `start-of-week` by the [[driver.common/start-of-week-offset]], which comesfrom [[driver/db-start-of-week]]."- ([driver day-of-week-honeysql-expr]+ ([driver :- :keyword+ day-of-week-honeysql-expr :- ::h2x/expr](adjust-day-of-week driver day-of-week-honeysql-expr (driver.common/start-of-week-offset driver)))- ([driver day-of-week-honeysql-expr offset]+ ([driver :- :keyword+ day-of-week-honeysql-expr :- ::h2x/expr+ offset :- :int](adjust-day-of-week driver day-of-week-honeysql-expr offset h2x/mod))- ([driver- day-of-week-honeysql-expr+ ([driver :- :keyword+ day-of-week-honeysql-expr :- ::h2x/exproffset :- :int- mod-fn :- [:=> [:cat any? any?] any?]]+ mod-fn :- [:=> [:cat ::h2x/expr ::h2x/expr] ::h2x/expr]](cond(inline? offset) (recur driver day-of-week-honeysql-expr (second offset) mod-fn)(zero? offset) day-of-week-honeysql-expr@@ -911,7 +916,7 @@`AS`).(field-source-table-aliases [:field 1 nil]) ; -> [\"public\" \"venues\"]"- [[_ opts id-or-name]]+ [[_ opts id-or-name] :- :mbql.clause/field](let [source-table (or (get opts driver-api/qp.add.source-table)(when (integer? id-or-name)(:table-id (driver-api/field (driver-api/metadata-provider) id-or-name))))]@@ -1516,7 +1521,9 @@(:name (driver-api/field (driver-api/metadata-provider) id-or-name))))](->honeysql driver (h2x/identifier :field-alias desired-alias))))- ([driver field-clause _unique-name-fn]+ ([driver :- :keyword+ field-clause :- vector?+ _unique-name-fn :- [:maybe ifn?]](sql.qp.deprecated/log-deprecation-warningdriver"metabase.driver.sql.query-processor/field-clause->alias with 3 args"@@ -1729,12 +1736,13 @@(mu/defn- generate-pattern"Generate pattern to match against in like clause. Lowercasing for case insensitive matching also happens here."- [driver- pre+ [driver :- :keyword+ pre :- [:maybe :string];; still typed by the deprecated legacy schema above; both go away with the MBQL 5 migration[type _ :as arg] :- #_{:clj-kondo/ignore [:deprecated-var]} LegacyStringValueOrFieldOrExpression- post- {:keys [case-sensitive] :or {case-sensitive true} :as _options}]+ post :- [:maybe :string]+ {:keys [case-sensitive] :or {case-sensitive true} :as _options}+ :- [:merge :metabase.lib.schema.common/options :metabase.lib.schema.filter/string-filter-options]](if (= :value type)(->> (update arg 2 #(cond-> (str pre (escape-like-pattern driver %) post)(not case-sensitive) u/lower-case-en))@@ -1757,9 +1765,25 @@(:base-type opts)):type/UUID))))+(mr/def ::compilable-expression+ "An argument [[->honeysql]] compiles in a filter: an MBQL 5 expression, a UUID, or a driver's own clause (tagged with a+ namespaced keyword, e.g. `:metabase.driver.sqlserver/cast`) wrapping one."+ [:or+ :metabase.lib.schema.expression/expression+ uuid?+ [:tuple [:= ::compiled] ::h2x/honeysql-expr]+ [:and+ vector?+ [:cat+ qualified-keyword?+ [:? [:or [:= {} {}] :metabase.lib.schema.common/options]]+ [:* [:schema [:or :string [:ref ::compilable-expression]]]]]]])+(mu/defn- maybe-cast-uuid-for-equality"For := and :!=. Comparing UUID fields against non-uuid values requires casting."- [driver field arg]+ [driver :- :keyword+ field :- ::compilable-expression+ arg :- ::compilable-expression](if (and (uuid-field? field);; If the arg is a uuid we are happy especially for joins (#46558)(not (uuid-field? arg))@@ -1774,7 +1798,8 @@(mu/defn maybe-cast-uuid-for-text-compare"For :contains, :starts-with, and :ends-with.Comparing UUID fields against with these operations requires casting as the right side will have `%` for `LIKE` operations."- [_driver field]+ [_driver :- :keyword+ field :- ::compilable-expression](if (uuid-field? field)[::cast-to-text {} field]field))@@ -2242,7 +2267,7 @@(mu/defn mbql->honeysql :- [:or :map [:tuple [:= :inline] :map]]"Build the HoneySQL form we will compile to SQL and execute."[driver :- :keyword- query :- :map]+ query :- ::qp.schema/any-query](if (:lib/type query)(binding [driver/*driver* driver](let [stages (preprocess driver query)]@@ -2266,7 +2291,7 @@"Transpile MBQL query into a native SQL statement. This is the `:sql` driver implementationof [[driver/mbql->native]] (actual multimethod definition is in [[metabase.driver.sql]]."[driver :- :keyword- outer-query :- :map]+ outer-query :- ::qp.schema/any-query](let [honeysql-form (mbql->honeysql driver outer-query)[sql & args] (format-honeysql driver honeysql-form)]{:query sql, :params args}))diff --git a/src/metabase/driver/sql/util.clj b/src/metabase/driver/sql/util.cljindex d0cb4711a94..3eae0aacec7 100644--- a/src/metabase/driver/sql/util.clj+++ b/src/metabase/driver/sql/util.clj@@ -28,7 +28,7 @@SQL. For HoneySQL forms, `Identifier` is converted to SQL automatically when it is compiled."[driver :- :keywordidentifier-type :- h2x/IdentifierType- & components]+ & components :- [:* {:min 1} [:maybe [:or :keyword :string [:fn h2x/identifier?]]]]](first(sql.qp.format/format-honeysql driver (apply h2x/identifier identifier-type components))))diff --git a/src/metabase/driver/sql_jdbc/actions.clj b/src/metabase/driver/sql_jdbc/actions.cljindex 912f2dd545c..8b51aaf7f32 100644--- a/src/metabase/driver/sql_jdbc/actions.clj+++ b/src/metabase/driver/sql_jdbc/actions.clj@@ -272,7 +272,8 @@(mu/defn- correct-columns-name :- [:maybe [:sequential driver-api/schema.actions.args.row]]"Ensure each rows have column name match with fields name.Some drivers like h2 have weird issue with casing."- [table-id rows :- [:sequential driver-api/schema.actions.args.row]]+ [table-id :- driver-api/schema.id.table+ rows :- [:sequential driver-api/schema.actions.args.row]](when (seq rows)(let [field-names (driver-api/cached-value[::correct-columns-name table-id]@@ -357,7 +358,9 @@:after nil}))))(mu/defn- model-row-delete! :- (result-schema [:map [:rows-deleted :int]])- [action context inputs]+ [action :- qualified-keyword?+ context :- driver-api/schema.actions.execution-context+ inputs :- [:sequential :metabase.legacy-mbql.schema/Query]](let [database (inputs->db inputs);; TODO it would be nice to make this 1 statement per table, instead of N.;; we can rely on the table lock instead of the nested row transactions.@@ -479,8 +482,9 @@;; TODO (Cam 2026-07-23) Update this stuff to use MBQL 5 instead of legacy MBQL(mu/defn- model-create! :- (result-schema [:map [:created-row driver-api/schema.actions.args.row]])- ;; the legacy MBQL query schema is deprecated, and goes away with the TODO above- [action context legacy-queries :- #_{:clj-kondo/ignore [:deprecated-var]} [:sequential driver-api/mbql.schema.Query]]+ [action :- qualified-keyword?+ context :- driver-api/schema.actions.execution-context+ legacy-queries :- [:sequential :metabase.legacy-mbql.schema/Query]](let [database (inputs->db legacy-queries);; TODO it would be nice to make this 1 statement per table, instead of N.;; we can rely on the table lock instead of the nested row transactions.@@ -550,7 +554,8 @@:xform (mapcat #(map (partial input-fn database (:table-id %)) (:rows %)))})))(mr/def ::table-row-input- [:map+ [:map {:closed true}+ [:database {:optional true} driver-api/schema.id.database][:table-id driver-api/schema.id.table][:row driver-api/schema.actions.args.row]])@@ -562,7 +567,9 @@:create-row row})(mu/defn- table-row-create!- [_action context inputs :- [:sequential ::table-row-input]]+ [_action :- qualified-keyword?+ context :- driver-api/schema.actions.execution-context+ inputs :- [:sequential ::table-row-input]](let [[errors results](batch-execution-by-table-id!{:row-action :model.row/create@@ -658,7 +665,9 @@{:status-code 400, :repeated-rows repeats}))))(mu/defn- table-row-delete!- [_action context inputs :- [:sequential ::table-row-input]]+ [_action :- qualified-keyword?+ context :- driver-api/schema.actions.execution-context+ inputs :- [:sequential ::table-row-input]](let [table-id->pk-keys (u/for-map [table-id (distinct (map :table-id inputs))](let [database (driver-api/cached-database-via-table-id table-id)field-name->id (table-id->pk-field-name->id (:id database) table-id)]@@ -738,7 +747,9 @@:update-row (apply dissoc row pk-names)}))(mu/defn- table-row-update!- [_action context inputs :- [:sequential ::table-row-input]]+ [_action :- qualified-keyword?+ context :- driver-api/schema.actions.execution-context+ inputs :- [:sequential ::table-row-input]](let [[errors results](batch-execution-by-table-id!{:inputs inputsdiff --git a/src/metabase/driver/sql_jdbc/connection.clj b/src/metabase/driver/sql_jdbc/connection.cljindex 8dd15d9b522..0317aa1ba3d 100644--- a/src/metabase/driver/sql_jdbc/connection.clj+++ b/src/metabase/driver/sql_jdbc/connection.clj@@ -20,6 +20,7 @@[metabase.util.log :as log][metabase.util.malli :as mu][metabase.util.performance :refer [get-in mapv select-keys]]+ [metabase.warehouses.schema :as warehouses.schema][potemkin :as p])(:import(com.mchange.v2.c3p0 C3P0ProxyConnection DataSources)@@ -310,7 +311,7 @@"Computes a hash value for the JDBC connection spec based on the effective connection details, for the purpose ofdetermining if details changed and therefore the existing connection pool needs to be invalidated.Uses [[driver.conn/effective-details]] to select the appropriate details for the current connection context."- [{driver :engine, :as database} :- [:maybe :map]]+ [{driver :engine, :as database} :- [:maybe ::warehouses.schema/database-or-metadata]](when (some? database)(hash (connection-details->spec driver (driver.conn/effective-details database)))))diff --git a/src/metabase/driver/sql_jdbc/execute.clj b/src/metabase/driver/sql_jdbc/execute.cljindex 006aee47b56..480b8e5b00e 100644--- a/src/metabase/driver/sql_jdbc/execute.clj+++ b/src/metabase/driver/sql_jdbc/execute.clj@@ -47,7 +47,7 @@(def ConnectionOptions"Malli schema for the options passed to [[do-with-connection-with-options]]."[:maybe- [:map+ [:map {:closed true};; a string like 'US/Pacific' or something like that.[:session-timezone {:optional true} [:maybe [:ref driver-api/schema.expression.temporal.timezone-id]]];; whether this Connection should NOT be read-only, e.g. for DDL stuff or inserting data or whatever.@@ -260,7 +260,7 @@(def ^:private DbOrIdOrSpec[:and- [:or :int :map]+ [:or :int driver-api/schema.metadata.database :metabase.lib.schema.common/database-details][:fn;; can't wrap a java.sql.Connection here because we're not;; responsible for its lifecycle and that means you can't use@@ -361,7 +361,7 @@deprecated [[sql-jdbc.execute.old/connection-with-timezone]] method."{:added "0.47.0"}[driver :- :keyword- db-or-id-or-spec :- [:or ::lib.schema.id/database :map]+ db-or-id-or-spec :- [:or ::lib.schema.id/database driver-api/schema.metadata.database :metabase.lib.schema.common/database-details]options :- ConnectionOptionsf :- fn?](binding [*connection-recursion-depth* (inc *connection-recursion-depth*)]@@ -398,8 +398,11 @@Connection."{:added "0.47.0"}[driver :- :keyword- db-or-id-or-spec- ^Connection conn :- (driver-api/instance-of-class Connection)+ db-or-id-or-spec :- [:or+ ::lib.schema.id/database+ driver-api/schema.metadata.database+ :metabase.lib.schema.common/database-details]+ ^Connection conn :- (driver-api/instance-of-class Connection){:keys [^String session-timezone write?], :as options} :- ConnectionOptions](when-let [db (cond;; id?diff --git a/src/metabase/driver/sql_jdbc/metadata.clj b/src/metabase/driver/sql_jdbc/metadata.cljindex dc6fae56425..7c81722bc8f 100644--- a/src/metabase/driver/sql_jdbc/metadata.clj+++ b/src/metabase/driver/sql_jdbc/metadata.clj@@ -5,6 +5,7 @@[metabase.driver-api.core :as driver-api][metabase.driver.sql-jdbc.execute :as sql-jdbc.execute][metabase.driver.sql-jdbc.sync.interface :as sql-jdbc.sync.interface]+ [metabase.query-processor.schema :as qp.schema][metabase.util.malli :as mu][metabase.util.performance :refer [mapv]]))@@ -14,7 +15,7 @@"Default implementation of [[metabase.driver/query-result-metadata]] for JDBC-based drivers. Gets metadata withoutactually running a query."([driver :- :keyword- query :- :map]+ query :- ::qp.schema/any-query](driver-api/with-qp-setup [query query](let [database (driver-api/database (driver-api/metadata-provider)){:keys [query params]} (driver-api/compile query)]@@ -23,7 +24,7 @@([driver :- :keyworddatabase :- driver-api/schema.metadata.database^String sql :- :string- params :- [:maybe [:sequential :any]]]+ params :- [:maybe [:sequential :metabase.lib.schema.common/field-value]]](sql-jdbc.execute/do-with-connection-with-optionsdriverdatabasediff --git a/src/metabase/driver/sql_jdbc/sync/describe_database.clj b/src/metabase/driver/sql_jdbc/sync/describe_database.cljindex 25ac8da4152..dc04caf52da 100644--- a/src/metabase/driver/sql_jdbc/sync/describe_database.clj+++ b/src/metabase/driver/sql_jdbc/sync/describe_database.clj@@ -15,7 +15,8 @@[metabase.util.honey-sql-2 :as h2x][metabase.util.log :as log][metabase.util.malli :as mu]- [metabase.util.performance :refer [get-in]])+ [metabase.util.performance :refer [get-in]]+ [metabase.warehouses.schema :as warehouses.schema])(:import(java.sql Connection DatabaseMetaData ResultSet)))@@ -311,7 +312,7 @@eager -- satisfying [[metabase.driver/do-with-resilient-connection]], which requires an eager `f` -- whilethe connection is opened lazily at reduction time (with a fresh connection, outside the resilient scope)."[driver :- :keyword- db-or-id-or-spec :- [:or :int :map]]+ db-or-id-or-spec :- [:or :int ::warehouses.schema/database]]{:tables(reify clojure.lang.IReduceInit(reduce [_this rf init]diff --git a/src/metabase/driver/sql_jdbc/sync/describe_table.clj b/src/metabase/driver/sql_jdbc/sync/describe_table.cljindex f897c0bab20..868a156f284 100644--- a/src/metabase/driver/sql_jdbc/sync/describe_table.clj+++ b/src/metabase/driver/sql_jdbc/sync/describe_table.clj@@ -466,7 +466,7 @@[:cat(lib.schema.common/instance-of-class Connection)[:map- {:description "Table"}+ {:closed true, :description "Table"}[:schema {:optional true} [:maybe :string]][:name :string]]]::driver/describe-fks.result]]diff --git a/src/metabase/driver/util.clj b/src/metabase/driver/util.cljindex b872be1d325..ea1cb7f05aa 100644--- a/src/metabase/driver/util.clj+++ b/src/metabase/driver/util.clj@@ -23,9 +23,8 @@[metabase.util.i18n :refer [deferred-tru trs]][metabase.util.log :as log][metabase.util.malli :as mu]- ;; ms/InstanceOf validates Toucan Database instances; lib.schema has no app-db instance schemas- ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.util.malli.schema :as ms]- [metabase.util.performance :refer [mapv empty? some]])+ [metabase.util.performance :refer [mapv empty? some]]+ [metabase.warehouses.schema :as warehouses.schema])(:import(java.io ByteArrayInputStream)(java.security KeyFactory KeyStore PrivateKey)@@ -295,13 +294,10 @@(This is cached for a second, so as to avoid repeated application DB calls if this function is called several timesover the duration of a single API request or sync operation.)"- [database-or-id :- [:or- {:error/message "Database or ID"}- [:map- [:engine [:or :keyword :string]]]- [:map- [:id ::lib.schema.id/database]]- ::lib.schema.id/database]]+ [database-or-id :- [:maybe [:or+ {:error/message "Database or ID"}+ ::lib.schema.id/database+ ::warehouses.schema/database-or-metadata]]](if-let [driver (:engine database-or-id)];; ensure we get the driver as a keyword (sometimes it's a String)(keyword driver)@@ -350,14 +346,10 @@[driver feature (mdb/unique-identifier) (:id database) (:updated-at database)])))));;; this can get called in post-select which doesn't always have ID-(mu/defn ensure-lib-database :- [:map- [:lib/type [:= :metadata/database]]]+(mu/defn ensure-lib-database :- [:map [:lib/type [:= :metadata/database]]]"Ensures the database is in Lib metadata format (SnakeHatingMap with kebab-case keys).If passed a Toucan2 instance, converts it. If already Lib metadata, returns as-is."- [database :- [:or- [:map- [:lib/type [:= :metadata/database]]]- (ms/InstanceOf :model/Database)]]+ [database :- ::warehouses.schema/database-or-metadata](if-not (:lib/type database)(lib-be/instance->metadata database :metadata/database)database))@@ -373,12 +365,7 @@and move on for now."[driver :- :keywordfeature :- :keyword- database :- [:maybe- [:or- ;; this can get called with an incomplete object in post-select- [:map- [:lib/type [:= :metadata/database]]]- (ms/InstanceOf :model/Database)]]]+ database :- [:maybe ::warehouses.schema/database-or-metadata]](let [database (some-> database ensure-lib-database)f (if *memoize-supports?* memoized-supports?* supports?*)](f driver feature database)))@@ -427,11 +414,7 @@(mu/defn features"Return a set of all features supported by `driver` with respect to `database`."[driver :- :keyword- database :- [:or- ;; this can get called in post-select which doesn't always have ID- [:map- [:lib/type [:= :metadata/database]]]- (ms/InstanceOf :model/Database)]]+ database :- ::warehouses.schema/database-or-metadata](let [database (ensure-lib-database database)](if *memoize-supports?*(memoized-features* driver database)@@ -483,13 +466,13 @@(-> {:name (str prop-name "-value"):type "textFile":treat-before-posting "base64"- :visible-if {(keyword (str prop-name "-options")) "uploaded"}}+ :visible-if {(str prop-name "-options") "uploaded"}}(dissoc :secret-kind)){:name (str prop-name "-path"):type "string":display-name (trs "File path"):placeholder (:placeholder conn-prop)- :visible-if {(keyword (str prop-name "-options")) "local"}}]))+ :visible-if {(str prop-name "-options") "local"}}]))(defn- ->str"Turns `x` into a String. If `x` a keyword, then `name` is used. Otherwise, `str` is called on it."@@ -560,14 +543,14 @@:type "text":placeholder placeholder:description (trs "Comma separated names of {0} that should appear in Metabase" (u/lower-case-en disp-name))- :visible-if {(keyword type-prop-nm) "inclusion"}+ :visible-if {type-prop-nm "inclusion"}:helper-text (trs "You can use patterns like \"auth*\" to match multiple {0}" (u/lower-case-en disp-name)):required true}{:name (str prop-name "-patterns"):type "text":placeholder placeholder:description (trs "Comma separated names of {0} that should NOT appear in Metabase" (u/lower-case-en disp-name))- :visible-if {(keyword type-prop-nm) "exclusion"}+ :visible-if {type-prop-nm "exclusion"}:helper-text (trs "You can use patterns like \"auth*\" to match multiple {0}" (u/lower-case-en disp-name)):required true}]))diff --git a/src/metabase/driver_api/core.clj b/src/metabase/driver_api/core.cljindex d419e27a2fc..4eb9375b3bf 100644--- a/src/metabase/driver_api/core.clj+++ b/src/metabase/driver_api/core.clj@@ -4,6 +4,7 @@(:refer-clojure :exclude [replace compile])(:require[metabase.actions.core :as actions]+ [metabase.actions.schema :as actions.schema][metabase.api.common :as api][metabase.app-db.core :as mdb][metabase.appearance.core :as appearance]@@ -240,6 +241,10 @@":metabase.actions.args/row":metabase.actions.args/row)+(def schema.actions.execution-context+ "::metabase.actions.schema/execution-context"+ ::actions.schema/execution-context)+(def schema.expression.temporal.timezone-id"::lib.schema.expression.temporal/timezone-id"::lib.schema.expression.temporal/timezone-id)diff --git a/src/metabase/eid_translation/util.clj b/src/metabase/eid_translation/util.cljindex 566c1014a73..7ff4ff6af32 100644--- a/src/metabase/eid_translation/util.clj+++ b/src/metabase/eid_translation/util.clj@@ -105,7 +105,8 @@:string[:map [:status eid-translation/Status]]]]"Given a model and a sequence of entity ids on that model, return a pairs of entity-id, id."- [api-name eids]+ [api-name :- [:or ApiName ApiModel]+ eids :- [:sequential :string]](let [model (->model api-name) ;; This lookup is safe because we've already validated the api-nameseid->id (into {} (case model:model/Action (eid-translation.db/action-ids-by-entity-ids eids)diff --git a/src/metabase/embedding/jwt.clj b/src/metabase/embedding/jwt.cljindex 9535e921f7b..8d5cb8f0ced 100644--- a/src/metabase/embedding/jwt.clj+++ b/src/metabase/embedding/jwt.clj@@ -25,6 +25,13 @@(when (= alg "none")(throw (Exception. (trs "JWT `alg` cannot be `none`."))))))+(defn- stringify-slug-keys+ "Normalize the `:params` and `:_embedding_params` slug maps of decoded `claims` back to string keys."+ [claims]+ (cond-> claims+ (map? (:params claims)) (update :params update-keys name)+ (map? (:_embedding_params claims)) (update :_embedding_params update-keys name)))+(defn unsign"Parse a \"signed\" (base-64 encoded) JWT and return a Clojure representation. Check that the signature isvalid (i.e., check that it was signed with `embedding-secret-key`) and it's otherwise a valid JWT (e.g., not@@ -33,12 +40,13 @@(when (seq message)(try(check-valid-alg message)- (jwt/unsign message- (or (embedding.settings/embedding-secret-key)- (throw (ex-info (tru "The embedding secret key has not been set.") {:status-code 400})))- ;; The library will reject tokens with a created at timestamp in the future, so to account for clock- ;; skew tell the library to allow for 60 seconds of leeway- {:leeway 60})+ (stringify-slug-keys+ (jwt/unsign message+ (or (embedding.settings/embedding-secret-key)+ (throw (ex-info (tru "The embedding secret key has not been set.") {:status-code 400})))+ ;; The library will reject tokens with a created at timestamp in the future, so to account for clock+ ;; skew tell the library to allow for 60 seconds of leeway+ {:leeway 60}));; if `jwt/unsign` throws an Exception rethrow it in a format that's friendlier to our API(catch Throwable e(throw (ex-info (ex-message e) {:status-code 400}))))))diff --git a/src/metabase/embedding/models/theme.clj b/src/metabase/embedding/models/theme.cljindex ea753dc1eb7..7ae6e14926c 100644--- a/src/metabase/embedding/models/theme.clj+++ b/src/metabase/embedding/models/theme.clj@@ -1,6 +1,7 @@(ns metabase.embedding.models.theme"Model for embedding themes for use in the embedding theme editor."(:require+ [metabase.embedding.schema][metabase.models.interface :as mi][metabase.models.serialization :as serdes][methodical.core :as methodical]diff --git a/src/metabase/embedding/schema.clj b/src/metabase/embedding/schema.cljindex 4d167f4c4fa..688a5618c00 100644--- a/src/metabase/embedding/schema.clj+++ b/src/metabase/embedding/schema.clj@@ -6,18 +6,14 @@(mr/def ::embedding-theme.settings"The `:settings` column of a EmbeddingTheme, decoded."- :map)+ ms/OpaqueJSONObject)(mr/def ::embedding-theme"A EmbeddingTheme as selected from the app DB: every column of `:embedding_theme`."- [:map {:closed true}- [:id ms/PositiveInt]- [:entity_id :string]- [:name :string]- [:settings ::embedding-theme.settings]- [:is_default :boolean]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]])+ [:merge+ ::embedding-theme.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::embedding-theme.update"What an update (or insert) of a EmbeddingTheme accepts: every column of `:embedding_theme` except `id`, all optional."diff --git a/src/metabase/embedding_rest/api/common.clj b/src/metabase/embedding_rest/api/common.cljindex dfff9f956f3..a6336df84b0 100644--- a/src/metabase/embedding_rest/api/common.clj+++ b/src/metabase/embedding_rest/api/common.clj@@ -15,6 +15,7 @@[metabase.notification.payload.core :as notification.payload][metabase.parameters.dashboard :as parameters.dashboard][metabase.parameters.params :as params]+ [metabase.parameters.schema :as parameters.schema][metabase.public-sharing-rest.api :as api.public][metabase.queries.core :as queries][metabase.query-processor.card :as qp.card]@@ -53,7 +54,28 @@(def ParsedQueryParams"Schema for [[QueryParams]] after the `:parameters` JSON blob has been decoded into real JSON scalars."- [:map-of [:or :keyword :string] [:maybe [:ref ::lib.schema.parameter/parameter.value]]])+ [:map-of :string [:maybe [:ref ::lib.schema.parameter/parameter.value]]])++(def ParamValue+ "A single dashboard/card parameter value, or a sequence of them."+ [:maybe [:ref ::lib.schema.parameter/parameter.value]])++(def SlugValueMap+ "A map of dashboard/card parameter slug to its value, slugs normalized to strings whether they came off a+ decoded JWT or the query string."+ [:map-of :string ParamValue])++(def ^:private ResourceId+ [:or ms/PositiveInt ms/NanoIdString])++(def UnsignedToken+ "The decoded, but not necessarily resource-id-translated, payload of an embedding JWT, whose other claims (`exp`, `iat`, ...) belong to the embedding application."+ [:map {:closed false, ::mr/deliberately-open true, :description "embedding JWT claims"}+ [:resource {:optional true} [:map {:closed true}+ [:question {:optional true} ResourceId]+ [:dashboard {:optional true} ResourceId]]]+ [:params {:optional true} SlugValueMap]+ [:_embedding_params {:optional true} SlugValueMap]])(comment;; load dynamic model resolution code... should already be loaded by [[metabase.core.init]] so this is mostly here for@@ -139,8 +161,8 @@"Given a pre-loaded `dashboard` entity and parameters map in the format `slug->value`, return a sequence ofparameters with `:id`s that can be passed to various functions in the [[metabase.dashboards-rest.api]] namespace suchas [[metabase.dashboards-rest.api/process-query-for-dashcard]]."- [dashboard :- :map- slug->value :- :map]+ [dashboard :- ::dashboards.schema/dashboard+ slug->value :- SlugValueMap](let [parameters (:parameters dashboard)slug->id (into {} (map (juxt :slug :id)) parameters)slug->type (into {} (map (juxt :slug :type)) parameters)]@@ -165,10 +187,10 @@booleans and boolean strings. To fix this issue we introduced another query string parameter `:parameters` whichcontains serialized JSON with parameter values. If this object cannot be found or parsed, we fallback to plain querystring parameters."- [query-params]+ [query-params :- QueryParams](let [parsed (when-let [parameters (some query-params [:parameters "parameters"])](try- (json/decode+kw parameters)+ (json/decode parameters)(catch Throwable _nil)))](when (and (some? parsed)@@ -176,25 +198,25 @@(throw (ex-info (tru "Invalid parameter values") {:status-code 400})))(or parsed query-params {})))-(mu/defn normalize-query-params :- [:map-of :keyword :any]+(mu/defn normalize-query-params :- [:map-of :string :any]"Take a map of `query-params` and make sure they're in the right format for the rest of our code. Our`wrap-keyword-params` middleware normally converts all query params keys to keywords, but only if they seem likeones that make sense as keywords. Some params, such as ones that start with a number, do not pass this test, and arenot automatically converted. Thus we must do it ourselves here to make sure things are done as we'd expect.Also, any param values that are blank strings should be parsed as nil, representing the absence of a value."- [query-params]+ [query-params :- [:or QueryParams ParsedQueryParams]](-> query-params- (update-keys keyword)+ (update-keys name)(update-vals (fn [v] (if (= v "") nil v)))))-(mu/defn validate-and-merge-params :- [:map-of :keyword :any]+(mu/defn validate-and-merge-params :- [:map-of :string :any]"Validate that the `token-params` passed in the JWT and the `user-params` (passed as part of the URL) are allowed, andthat ones that are required are specified by checking them against a Card or Dashboard's `object-embedding-params`(the object's value of `:embedding_params`). Throws a 400 if any of the checks fail. If all checks are successful,returns a *merged* parameters map."[object-embedding-params :- ms/EmbeddingParams- token-params :- [:map-of :keyword :any]- user-params :- [:map-of :keyword :any]]+ token-params :- SlugValueMap+ user-params :- SlugValueMap](check-param-sets object-embedding-params(m/filter-vals valid-param-value? token-params)(m/filter-vals valid-param-value? user-params))@@ -207,7 +229,11 @@v)))))(mu/defn- param-values-merged-params :- [:map-of ms/NonBlankString :any]- [id->slug slug->id embedding-params token-params id-query-params]+ [id->slug :- [:map-of ms/NonBlankString ms/NonBlankString]+ slug->id :- [:map-of ms/NonBlankString ms/NonBlankString]+ embedding-params :- ms/EmbeddingParams+ token-params :- SlugValueMap+ id-query-params :- [:or QueryParams ParsedQueryParams]](let [slug-query-params (into {}(for [[id v] id-query-params][(or (get id->slug (name id))@@ -228,12 +254,21 @@;;; ---------------------------------------------- Other Param Util Fns ----------------------------------------------+(defn- param-id->slug+ "Build an id->slug lookup from `parameters`, skipping any without a `:slug` — such a parameter can never be+ looked up by slug, and a nil slug would wreck `set/map-invert`."+ [parameters]+ (into {} (comp (filter :slug) (map (juxt :id :slug))) parameters))+(defn- locked-slug->value"The `\"locked\"` parameter values carried by the signed JWT. These are supplied by the embedding server ratherthan the client, and are passed to the parameter value lookups as constraints so the values offered for theenabled parameters match the rows the locked values select."[embedding-params token-params]- (into {} (filter (fn [[slug _value]] (= (get embedding-params (name slug)) "locked"))) token-params))+ (into {}+ (comp (filter (fn [[slug _value]] (= (get embedding-params (name slug)) "locked")))+ (map (fn [[slug value]] [(name slug) value])))+ token-params))(defn- enabled-param-slugs"The set of param slugs (as keywords) from `dashboard-or-card-params` that may be exposed to embed viewers: only@@ -246,19 +281,19 @@dashboard-or-card-params))(mu/defn- enabled-params- "Keep only the `:parameters` of `dashboard-or-card` whose slug is listed as `enabled` in the `embedding-params`- whitelist, so the frontend doesn't display widgets for params (`disabled`, `locked`, or unlisted) the user can't- set."- [dashboard-or-card embedding-params :- ms/EmbeddingParams]- (let [param-slugs-to-keep (enabled-param-slugs (:parameters dashboard-or-card) embedding-params)]- (update dashboard-or-card :parameters (partial filter #(contains? param-slugs-to-keep (keyword (:slug %)))))))+ "Keep only the `parameters` whose slug is listed as `enabled` in the `embedding-params` whitelist, so the frontend+ doesn't display widgets for params (`disabled`, `locked`, or unlisted) the user can't set."+ [parameters :- [:maybe [:sequential ::parameters.schema/parameter]]+ embedding-params :- ms/EmbeddingParams]+ (let [param-slugs-to-keep (enabled-param-slugs parameters embedding-params)]+ (filter #(contains? param-slugs-to-keep (keyword (:slug %))) parameters)))(defn- remove-token-parameters"Removes any parameters with slugs matching keys provided in `token-params`, as these should not be exposed to theuser."[dashboard-or-card token-params](let [token-slugs (set (keys token-params))]- (update dashboard-or-card :parameters (partial remove #(contains? token-slugs (keyword (:slug %)))))))+ (update dashboard-or-card :parameters (partial remove #(contains? token-slugs (:slug %))))))(defn- substitute-token-parameters-in-text"For any dashboard parameters with slugs matching keys provided in `token-params`, substitute their values from the@@ -269,7 +304,7 @@dashcards (:dashcards dashboard)params-with-values (reduce(fn [acc param]- (if-let [value (get token-params (keyword (:slug param)))]+ (if-let [value (get token-params (:slug param))](conj acc (assoc param :value value))acc))[]@@ -291,10 +326,11 @@[:value :any]]]]"Adds `value` to parameters with `slug` matching a key in `merged-slug->value` and removes parameters without a`value`."- [parameters slug->value]+ [parameters :- [:sequential ::parameters.schema/parameter]+ slug->value :- SlugValueMap](when (seq parameters)(for [param parameters- :let [slug (keyword (:slug param))+ :let [slug (:slug param)value (get slug->value slug);; operator parameters expect a sequence of values so if we get a lone value (e.g. from a single URL;; query parameter) wrap it in a sequence@@ -324,7 +360,7 @@(-> (api.public/public-card card-id :enable-embedding? enable-embedding?)api.public/combine-parameters-and-template-tags(remove-token-parameters token-params)- (enabled-params resolved-embedding-params)+ (update :parameters enabled-params resolved-embedding-params)api.public/keep-param-fields-for-parameters(assoc :embedding_params resolved-embedding-params))))@@ -363,13 +399,13 @@(defn- tile-slug->value[object-parameters parameter-values]- (let [id->slug (into {} (map (juxt :id :slug)) object-parameters)]+ (let [id->slug (param-id->slug object-parameters)](into {}(map (fn [{:keys [id value]}]- [(keyword (or (get id->slug id)- (throw (ex-info (tru "Invalid query params: could not determine slug for parameter with ID {0}"- (pr-str id))- {:status-code 400}))))+ [(or (get id->slug id)+ (throw (ex-info (tru "Invalid query params: could not determine slug for parameter with ID {0}"+ (pr-str id))+ {:status-code 400})))value]))parameter-values)))@@ -427,7 +463,10 @@(mu/defn dashboard-for-unsigned-token :- ::dashboards.schema/dashboard"Return the info needed for embedding about Dashboard specified in `token`."- [unsigned-token & {:keys [embedding-params enable-embedding?]}]+ [unsigned-token :- UnsignedToken+ & {:keys [embedding-params enable-embedding?]} :- [:maybe [:map {:closed true}+ [:embedding-params {:optional true} [:maybe ms/EmbeddingParams]]+ [:enable-embedding? {:optional true} [:maybe :boolean]]]]](let [dashboard-id (unsigned-token->dashboard-id unsigned-token)embedding-params (or embedding-params(embedding-rest.db/dashboard-embedding-params dashboard-id))@@ -436,7 +475,7 @@(substitute-token-parameters-in-text token-params)(remove-locked-parameters embedding-params)(remove-token-parameters token-params)- (enabled-params embedding-params)+ (update :parameters enabled-params embedding-params)api.public/keep-param-fields-for-parameters)))(defn- get-embed-dashboard-context@@ -488,7 +527,7 @@(let [slug-token-params (embed/get-in-unsigned-token-or-throw unsigned-token [:params])parameters (or (seq (:parameters card))(queries/card-template-tag-parameters card))- id->slug (into {} (map (juxt :id :slug)) parameters)+ id->slug (param-id->slug parameters)slug->id (set/map-invert id->slug)searched-param-slug (get id->slug param-key)embedding-params (:embedding_params card)]@@ -497,7 +536,7 @@(throw (ex-info (tru "Cannot search for values: {0} is not an enabled parameter."(pr-str searched-param-slug)){:status-code 400})))- (when (get slug-token-params (keyword searched-param-slug))+ (when (get slug-token-params searched-param-slug)(throw (ex-info (tru "You can''t specify a value for {0} if it''s already set in the JWT." (pr-str searched-param-slug)){:status-code 400})))(try@@ -535,7 +574,7 @@(let [slug-token-params (embed/get-in-unsigned-token-or-throw unsigned-token [:params])parameters (or (seq (:parameters card))(queries/card-template-tag-parameters card))- id->slug (into {} (map (juxt :id :slug)) parameters)+ id->slug (param-id->slug parameters)slug->id (set/map-invert id->slug)searched-param-slug (get id->slug param-key)embedding-params (:embedding_params card)]@@ -544,7 +583,7 @@(throw (ex-info (tru "Cannot get remapped value for parameter: {0} is not an enabled parameter."(pr-str searched-param-slug)){:status-code 400})))- (when (get slug-token-params (keyword searched-param-slug))+ (when (get slug-token-params searched-param-slug)(throw (ex-info (tru "You can''t specify a value for {0} if it''s already set in the JWT."(pr-str searched-param-slug)){:status-code 400})))@@ -597,7 +636,7 @@published-embedding-params(update-keys (get unsigned-token :_embedding_params) name))published-embedding-params)- id->slug (into {} (map (juxt :id :slug)) parameters)+ id->slug (param-id->slug parameters)slug->id (set/map-invert id->slug)searched-param-slug (get id->slug searched-param-id)](try@@ -605,7 +644,7 @@(when-not (= (get embedding-params searched-param-slug) "enabled")(throw (ex-info (tru "Cannot search for values: {0} is not an enabled parameter." (pr-str searched-param-slug)){:status-code 400})))- (when (get slug-token-params (keyword searched-param-slug))+ (when (get slug-token-params searched-param-slug)(throw (ex-info (tru "You can''t specify a value for {0} if it''s already set in the JWT." (pr-str searched-param-slug)){:status-code 400})));; ok, at this point we can run the query@@ -643,7 +682,7 @@_ (when-not preview (check-embedding-enabled-for-dashboard dashboard))slug-token-params (embed/get-in-unsigned-token-or-throw unsigned-token [:params])parameters (:parameters dashboard)- id->slug (into {} (map (juxt :id :slug)) parameters)+ id->slug (param-id->slug parameters)slug->id (set/map-invert id->slug)published-embedding-params (:embedding_params dashboard);; when previewing an embed, embedding-params should come from the token,@@ -663,7 +702,7 @@(when (not= (get embedding-params param-slug) "enabled")(throw (ex-info (tru "Cannot get remapped value for parameter: {0} is not an enabled parameter." (pr-str param-slug)){:status-code 400})))- (when (get slug-token-params (keyword param-slug))+ (when (get slug-token-params param-slug)(throw (ex-info (tru "You can''t specify a value for {0} if it''s already set in the JWT." (pr-str param-slug)){:status-code 400})))(let [constraints (-> (param-values-merged-params id->slug slug->id embedding-params slug-token-params {})diff --git a/src/metabase/embedding_rest/api/embed.clj b/src/metabase/embedding_rest/api/embed.cljindex e46c9bba399..ab9a0bd4c23 100644--- a/src/metabase/embedding_rest/api/embed.clj+++ b/src/metabase/embedding_rest/api/embed.clj@@ -32,17 +32,22 @@[metabase.tiles.api :as api.tiles][metabase.util :as u][metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms][ring.util.codec :as codec]))(set! *warn-on-reflection* true)-(def ^:private ResourceId [:or ms/PositiveInt ms/NanoIdString])-(def ^:private Token [:map- [:resource [:map- [:question {:optional true} ResourceId]- [:dashboard {:optional true} ResourceId]]]- [:params :any]])+(def ^:private ResourceId+ "An id or entity_id identifying a Card or Dashboard in a JWT `:resource`."+ [:or ms/PositiveInt ms/NanoIdString])+(def ^:private Token+ "An embedding JWT payload naming its `:resource` and `:params`, whose other claims (`exp`, `iat`, ...) belong to the embedding application."+ [:map {:closed false, ::mr/deliberately-open true, :description "embedding JWT claims"}+ [:resource [:map {:closed true}+ [:question {:optional true} ResourceId]+ [:dashboard {:optional true} ResourceId]]]+ [:params api.embed.common/SlugValueMap]])(defn- conditional-update-in"If there's a value at `path`, apply `f`, otherwise return `m`."diff --git a/src/metabase/explorations/db.clj b/src/metabase/explorations/db.cljindex 0188a9194eb..316da830ffd 100644--- a/src/metabase/explorations/db.clj+++ b/src/metabase/explorations/db.clj@@ -8,6 +8,7 @@[metabase.collections.models.collection :as collection][metabase.documents.schema :as documents.schema][metabase.explorations.schema :as explorations.schema]+ [metabase.interestingness.chart.types :as chart.types][metabase.lib.schema.id :as lib.schema.id][metabase.queries.core :as queries][metabase.queries.schema :as queries.schema]@@ -304,8 +305,8 @@[blocks :- (let [row [:map {:closed true}[:id {:optional true} ms/PositiveInt][:exploration_thread_id {:optional true} [:maybe ms/PositiveInt]]- [:metrics {:optional true} [:maybe [:sequential :map]]]- [:dimensions {:optional true} [:maybe [:sequential :map]]]+ [:metrics {:optional true} [:maybe [:sequential ::explorations.schema/exploration-block.metric]]]+ [:dimensions {:optional true} [:maybe [:sequential ::explorations.schema/exploration-block.dimension]]][:position {:optional true} [:maybe :int]][:created_at {:optional true} [:maybe ms/TemporalInstant]][:updated_at {:optional true} [:maybe ms/TemporalInstant]]]]@@ -573,7 +574,7 @@[:created_at {:optional true} [:maybe ms/TemporalInstant]][:interestingness_score {:optional true} [:maybe number?]][:contextual_interestingness_score {:optional true} [:maybe number?]]- [:chart_stats {:optional true} [:maybe :map]]+ [:chart_stats {:optional true} [:maybe ::chart.types/chart-stats]][:metric_description {:optional true} [:maybe :string]][:chart_description {:optional true} [:maybe :string]]]](t2/insert! :model/ExplorationQueryResult query-result))diff --git a/src/metabase/explorations/models/exploration.clj b/src/metabase/explorations/models/exploration.cljindex 476eaf3c042..fd683793c1e 100644--- a/src/metabase/explorations/models/exploration.clj+++ b/src/metabase/explorations/models/exploration.clj@@ -2,6 +2,7 @@(:require[clojure.string :as str][metabase.explorations.db :as explorations.db]+ [metabase.explorations.schema][metabase.models.interface :as mi][metabase.search.spec :as search.spec][metabase.util.i18n :refer [deferred-tru]]diff --git a/src/metabase/explorations/models/exploration_block.clj b/src/metabase/explorations/models/exploration_block.cljindex 503fedf73b1..79da67b0a2b 100644--- a/src/metabase/explorations/models/exploration_block.clj+++ b/src/metabase/explorations/models/exploration_block.clj@@ -12,6 +12,7 @@[metabase.explorations.db :as explorations.db][metabase.lib.core :as lib][metabase.lib.schema.parameter :as lib.schema.parameter]+ [metabase.metrics.core :as metrics][metabase.models.interface :as mi][methodical.core :as methodical][toucan2.core :as t2]))@@ -59,9 +60,25 @@explore-filters)))))metrics)))+(defn- normalize-dimension-mappings+ "Normalize the `:dimension_mappings` of each metric selection read back from JSON."+ [metrics]+ (when metrics+ (mapv (fn [metric]+ (cond-> metric+ (seq (:dimension_mappings metric))+ (update :dimension_mappings+ (fn [mappings]+ (mapv (fn [mapping]+ (cond-> mapping+ (:target mapping) (update :target metrics/normalize-target-ref)+ (:type mapping) (update :type keyword)))+ mappings)))))+ metrics)))+(def ^:private transform-metrics{:in (:in mi/transform-json)- :out (comp normalize-explore-filters (:out mi/transform-json))})+ :out (comp normalize-dimension-mappings normalize-explore-filters (:out mi/transform-json))})(t2/deftransforms :model/ExplorationBlock{:metrics transform-metricsdiff --git a/src/metabase/explorations/schema.clj b/src/metabase/explorations/schema.cljindex c24563cd223..b7c2021a693 100644--- a/src/metabase/explorations/schema.clj+++ b/src/metabase/explorations/schema.clj@@ -1,24 +1,19 @@(ns metabase.explorations.schema"Malli schemas for the explorations module."(:require+ [metabase.interestingness.chart.types :as chart.types]+ [metabase.lib.schema :as lib.schema][metabase.lib.schema.id :as lib.schema.id]+ [metabase.metrics.core :as metrics][metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms]))(mr/def ::exploration"A Exploration as selected from the app DB: every column of `:exploration`."- [:map {:closed true}- [:id ms/PositiveInt]- [:name :string]- [:description [:maybe :string]]- [:creator_id ::lib.schema.id/user]- [:collection_id [:maybe ::lib.schema.id/collection]]- [:archived :boolean]- [:archived_directly :boolean]- [:collection_position [:maybe :int]]- [:entity_id [:maybe :string]]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]])+ [:merge+ ::exploration.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::exploration.update"What an update (or insert) of a Exploration accepts: every column of `:exploration` except `id`, all optional."@@ -34,24 +29,44 @@[:created_at {:optional true} [:maybe ms/TemporalInstant]][:updated_at {:optional true} [:maybe ms/TemporalInstant]]])+(mr/def ::exploration-block.metric.explore-filter+ "One entry of a block metric selection's `:explore_filters` vector, decoded."+ [:multi {:dispatch :operator}+ ["=" [:map {:closed true}+ [:operator [:= "="]]+ [:field_ref [:ref :metabase.lib.schema.parameter/dimension.target]]+ [:value [:maybe [:or :string number? :boolean]]]+ [:display_value ms/NonBlankString]+ [:dimension_name {:optional true} [:maybe :string]]]]+ ["between" [:map {:closed true}+ [:operator [:= "between"]]+ [:field_ref [:ref :metabase.lib.schema.parameter/dimension.target]]+ [:values [:tuple [:maybe [:or :string number? :boolean]]+ [:maybe [:or :string number? :boolean]]]]+ [:display_value ms/NonBlankString]+ [:dimension_name {:optional true} [:maybe :string]]]]])+(mr/def ::exploration-block.metric"One entry of the `:metrics` column of a ExplorationBlock, decoded."- :map)+ [:map {:closed true}+ [:card_id ms/PositiveInt]+ [:dimension_mappings {:optional true} [:maybe [:sequential ::metrics/dimension-mapping]]]+ [:explore_filters {:optional true} [:maybe [:sequential ::exploration-block.metric.explore-filter]]]])(mr/def ::exploration-block.dimension"One entry of the `:dimensions` column of a ExplorationBlock, decoded."- :map)+ [:map {:closed true}+ [:dimension-id ms/UUIDString]+ [:display-name {:optional true} [:maybe :string]]+ [:effective-type {:optional true} [:maybe :string]]+ [:semantic-type {:optional true} [:maybe :string]]])(mr/def ::exploration-block"A ExplorationBlock as selected from the app DB: every column of `:exploration_block`."- [:map {:closed true}- [:id ms/PositiveInt]- [:exploration_thread_id ms/PositiveInt]- [:metrics [:maybe [:sequential ::exploration-block.metric]]]- [:dimensions [:maybe [:sequential ::exploration-block.dimension]]]- [:position :int]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]])+ [:merge+ ::exploration-block.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::exploration-block.update"What an update (or insert) of a ExplorationBlock accepts: every column of `:exploration_block` except `id`, all optional."@@ -65,18 +80,10 @@(mr/def ::exploration-page"A ExplorationPage as selected from the app DB: every column of `:exploration_page`."- [:map {:closed true}- [:id ms/PositiveInt]- [:entity_id [:maybe :string]]- [:exploration_block_id ms/PositiveInt]- [:card_id ::lib.schema.id/card]- [:dimension_id :string]- [:query_type [:or :keyword :string]]- [:position :int]- [:starred :boolean]- [:hidden :boolean]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]])+ [:merge+ ::exploration-page.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::exploration-page.update"What an update (or insert) of a ExplorationPage accepts: every column of `:exploration_page` except `id`, all optional."@@ -94,45 +101,29 @@(mr/def ::exploration-query.visualization-settings"The `:visualization_settings` column of a ExplorationQuery, decoded."- :map)+ ms/VisualizationSettings)(mr/def ::exploration-query.dataset-query"The `:dataset_query` column of a ExplorationQuery, decoded."- :map)+ ::lib.schema/query)(mr/def ::exploration-query.params"The `:params` column of a ExplorationQuery, decoded."- :map)+ [:map {:closed true}+ [:segment_id {:optional true} [:maybe ms/PositiveInt]]+ [:k {:optional true} [:maybe :int]]+ [:value_index {:optional true} [:maybe nat-int?]]])(mr/def ::exploration-query.data-access-token"The `:data_access_token` column of a ExplorationQuery, decoded."- :map)+ ::exploration-thread.data-access-token)(mr/def ::exploration-query"A ExplorationQuery as selected from the app DB: every column of `:exploration_query`."- [:map {:closed true}- [:id ms/PositiveInt]- [:exploration_thread_id ms/PositiveInt]- [:page_id ms/PositiveInt]- [:name [:maybe :string]]- [:card_id ::lib.schema.id/card]- [:database_id ::lib.schema.id/database]- [:segment_id [:maybe ::lib.schema.id/segment]]- [:dimension_id :string]- [:query_type [:or :keyword :string]]- [:display [:maybe [:or :keyword :string]]]- [:visualization_settings [:maybe ::exploration-query.visualization-settings]]- [:dataset_query [:maybe ::exploration-query.dataset-query]]- [:params [:maybe ::exploration-query.params]]- [:position :int]- [:status [:or :keyword :string]]- [:error_message [:maybe :string]]- [:started_at [:maybe ms/TemporalInstant]]- [:finished_at [:maybe ms/TemporalInstant]]- [:entity_id [:maybe :string]]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]- [:data_access_token [:maybe ::exploration-query.data-access-token]]])+ [:merge+ ::exploration-query.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::exploration-query.update"What an update (or insert) of a ExplorationQuery accepts: every column of `:exploration_query` except `id`, all optional."@@ -161,20 +152,14 @@(mr/def ::exploration-query-result.chart-stats"The `:chart_stats` column of a ExplorationQueryResult, decoded."- :map)+ ::chart.types/chart-stats)(mr/def ::exploration-query-result"A ExplorationQueryResult as selected from the app DB: every column of `:exploration_query_result`."- [:map {:closed true}- [:id ms/PositiveInt]- [:exploration_query_id ms/PositiveInt]- [:stored_result_id ms/PositiveInt]- [:created_at ms/TemporalInstant]- [:interestingness_score [:maybe number?]]- [:contextual_interestingness_score [:maybe number?]]- [:chart_stats [:maybe ::exploration-query-result.chart-stats]]- [:metric_description [:maybe :string]]- [:chart_description [:maybe :string]]])+ [:merge+ ::exploration-query-result.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::exploration-query-result.update"What an update (or insert) of a ExplorationQueryResult accepts: every column of `:exploration_query_result` except `id`, all optional."@@ -188,33 +173,69 @@[:metric_description {:optional true} [:maybe :string]][:chart_description {:optional true} [:maybe :string]]])+(mr/def ::exploration-thread.query-plan-transcript.plan-item+ "One entry of the `:plan` a query planner emits."+ [:map {:closed true}+ [:block_id ms/PositiveInt]+ [:metric_id ms/PositiveInt]+ [:dimension_id ms/UUIDString]+ [:variant :string]+ [:params {:optional true} [:map {:closed true}+ [:segment_id {:optional true} ms/PositiveInt]+ [:k {:optional true} :int]]]+ [:rationale {:optional true} [:maybe :string]]])++(mr/def ::exploration-thread.query-plan-transcript.planner-transcript+ "The planner-implementation-specific `:transcript` a query planner emits (see+ `metabase.explorations.query-plan.planner/plan!`); shape is per-implementation, only `:mechanical` exists today."+ [:or+ [:map {:closed true}+ [:reason :string]+ [:n-blocks :int]]+ [:map {:closed true}+ [:strategy :string]+ [:n-items :int]+ [:n-blocks :int]]])++(mr/def ::exploration-thread.query-plan-transcript.body+ "The `:transcript` entry of a query-plan transcript, decoded."+ [:map {:closed true}+ [:outcome {:optional true} [:maybe [:enum :ok :failed :skip-not-applicable]]]+ [:rationale {:optional true} [:maybe :string]]+ [:plan {:optional true} [:maybe [:sequential ::exploration-thread.query-plan-transcript.plan-item]]]+ [:final-errors {:optional true} [:maybe [:sequential :string]]]+ [:planner {:optional true} [:maybe ::exploration-thread.query-plan-transcript.planner-transcript]]])+(mr/def ::exploration-thread.query-plan-transcript"The `:query_plan_transcript` column of a ExplorationThread, decoded."- :map)+ [:map {:closed true}+ [:generated-at {:optional true} [:maybe :string]]+ [:thread-id {:optional true} [:maybe ms/PositiveInt]]+ [:planner {:optional true} [:maybe :keyword]]+ [:outcome {:optional true} [:maybe [:enum :ok :skip-empty :failed :error]]]+ [:rows-count {:optional true} [:maybe :int]]+ [:note {:optional true} [:maybe :keyword]]+ [:error {:optional true} [:maybe :string]]+ [:transcript {:optional true} [:maybe ::exploration-thread.query-plan-transcript.body]]])++(def ^:private DataAccessToken+ "A digested effective-data-access token: sandbox/impersonation/routing fingerprints, keyed by table or database+ id."+ [:map {:closed true}+ [:sandbox {:optional true} [:map-of ::lib.schema.id/table :string]]+ [:impersonation {:optional true} [:map-of ::lib.schema.id/database :string]]+ [:routing {:optional true} [:map-of ::lib.schema.id/database :string]]])(mr/def ::exploration-thread.data-access-token"The `:data_access_token` column of a ExplorationThread, decoded."- :map)+ DataAccessToken)(mr/def ::exploration-thread"A ExplorationThread as selected from the app DB: every column of `:exploration_thread`."- [:map {:closed true}- [:id ms/PositiveInt]- [:exploration_id ms/PositiveInt]- [:name [:maybe :string]]- [:prompt [:maybe :string]]- [:position :int]- [:source_page_id [:maybe ms/PositiveInt]]- [:started_at [:maybe ms/TemporalInstant]]- [:entity_id [:maybe :string]]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]- [:completed_at [:maybe ms/TemporalInstant]]- [:analysis_started_at [:maybe ms/TemporalInstant]]- [:query_plan_started_at [:maybe ms/TemporalInstant]]- [:query_plan_transcript [:maybe ::exploration-thread.query-plan-transcript]]- [:canceled_at [:maybe ms/TemporalInstant]]- [:data_access_token [:maybe ::exploration-thread.data-access-token]]])+ [:merge+ ::exploration-thread.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::exploration-thread.update"What an update (or insert) of a ExplorationThread accepts: every column of `:exploration_thread` except `id`, all optional."@@ -237,13 +258,10 @@(mr/def ::exploration-thread-timeline"A ExplorationThreadTimeline as selected from the app DB: every column of `:exploration_thread_timeline`."- [:map {:closed true}- [:id ms/PositiveInt]- [:exploration_thread_id ms/PositiveInt]- [:timeline_id ms/PositiveInt]- [:position :int]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]])+ [:merge+ ::exploration-thread-timeline.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::exploration-thread-timeline.update"What an update (or insert) of a ExplorationThreadTimeline accepts: every column of `:exploration_thread_timeline` except `id`, all optional."diff --git a/src/metabase/formatter/impl.clj b/src/metabase/formatter/impl.cljindex 36559d54dd0..367c8ef279d 100644--- a/src/metabase/formatter/impl.clj+++ b/src/metabase/formatter/impl.clj@@ -216,11 +216,15 @@(mu/defn format-scalar-number :- (ms/InstanceOfClass NumericWrapper)"Format a number `n` and return it as a NumericWrapper; this type is used to do special formatting in other`pulse.render` namespaces."- ([n :- number?]- (map->NumericWrapper {:num-str (cl-format nil (if (integer? n) "~:d" "~,2f") n)- :num-value n}))-- ([value column viz-settings]+ ([n :- [:maybe number?]]+ (map->NumericWrapper (if n+ {:num-str (cl-format nil (if (integer? n) "~:d" "~,2f") n)+ :num-value n}+ {:num-str "" :num-value nil})))++ ([value :- number?+ column :- [:or :metabase.legacy-mbql.schema/legacy-column-metadata :metabase.lib.schema.metadata/lib-or-legacy-column]+ viz-settings :- [:maybe ms/VisualizationSettings]](let [fmttr (number-formatter column viz-settings true)](fmttr value))))@@ -287,14 +291,19 @@(mu/defn create-formatter"Create a formatter for a column based on its timezone, column metadata, and visualization-settings"- ([timezone-id :- [:maybe :string] col visualization-settings]+ ([timezone-id :- [:maybe :string]+ col :- [:maybe [:or :metabase.legacy-mbql.schema/legacy-column-metadata :metabase.lib.schema.metadata/lib-or-legacy-column]]+ visualization-settings :- [:maybe ms/VisualizationSettings]](create-formatter timezone-id col visualization-settings true))- ([timezone-id :- [:maybe :string] col visualization-settings apply-formatting?]+ ([timezone-id :- [:maybe :string]+ col :- [:maybe [:or :metabase.legacy-mbql.schema/legacy-column-metadata :metabase.lib.schema.metadata/lib-or-legacy-column]]+ visualization-settings :- [:maybe ms/VisualizationSettings]+ apply-formatting? :- :boolean](cond;; for numbers, return a format function that has already computed the differences.;; todo: do the same for temporal strings(and apply-formatting?- #_{:clj-kondo/ignore [:deprecated-var]} (types/temporal-field? col)) ; legacy usage -- do not use going forward+ #_{:clj-kondo/ignore [:deprecated-var]} (types/temporal-field? (select-keys col [:base_type :effective_type]))) ; legacy usage -- do not use going forward(datetime/make-temporal-str-formatter timezone-id col visualization-settings)(and apply-formatting? (isa? (:semantic_type col) :type/Coordinate))@@ -351,11 +360,11 @@"Row/col/measure value formatters for a pivot export, keyed by `:row-formatters`/`:col-formatters`/`:val-formatters`.Shared by the CSV export and static-viz pivot render paths. `row-indexes`/`col-indexes`/`val-indexes` are columnindexes into `columns`."- [columns :- [:sequential :map]+ [columns :- [:sequential [:or :metabase.legacy-mbql.schema/legacy-column-metadata :metabase.lib.schema.metadata/lib-or-legacy-column]]row-indexes :- [:maybe [:sequential :int]]col-indexes :- [:maybe [:sequential :int]]val-indexes :- [:maybe [:sequential :int]]- settings :- [:maybe :map]+ settings :- [:maybe ms/VisualizationSettings]timezone :- [:maybe :string]format-rows? :- :boolean]{:row-formatters (create-formatters columns row-indexes timezone settings format-rows?)diff --git a/src/metabase/glossary/models/glossary.clj b/src/metabase/glossary/models/glossary.cljindex 7d52bc1221d..abc67dc15b4 100644--- a/src/metabase/glossary/models/glossary.clj+++ b/src/metabase/glossary/models/glossary.clj@@ -1,6 +1,7 @@(ns metabase.glossary.models.glossary(:require[metabase.glossary.db :as glossary.db]+ [metabase.glossary.schema][metabase.models.serialization :as serdes][metabase.util.malli.schema :as ms][methodical.core :as methodical]diff --git a/src/metabase/glossary/schema.clj b/src/metabase/glossary/schema.cljindex 906f30f9523..03863b693ba 100644--- a/src/metabase/glossary/schema.clj+++ b/src/metabase/glossary/schema.clj@@ -7,13 +7,10 @@(mr/def ::glossary"A Glossary as selected from the app DB: every column of `:glossary`."- [:map {:closed true}- [:id ms/PositiveInt]- [:term :string]- [:definition :string]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]- [:creator_id ::lib.schema.id/user]])+ [:merge+ ::glossary.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::glossary.update"What an update (or insert) of a Glossary accepts: every column of `:glossary` except `id`, all optional."diff --git a/src/metabase/graph/core.cljc b/src/metabase/graph/core.cljcindex 3e4b3c84c6f..d1270027431 100644--- a/src/metabase/graph/core.cljc+++ b/src/metabase/graph/core.cljc@@ -77,9 +77,16 @@{:error/message "Valid Graph instance"}#'graph?])-(mr/def ::node :any)+(mr/def ::node+ "A graph node key, whose shape each `Graph` implementation owns (an entity ID, an `[entity-type entity-id]` pair, ...)."+ [:schema {::mr/deliberately-open true, :description "any hashable graph node key"} :any])-(mr/def ::child-map [:map-of ::node [:set ::node]])+(mr/def ::child-map+ "A `{parent #{child}}` map keyed by the node values its `Graph` implementation owns."+ [:map-of {::mr/deliberately-open true, :description "keyed by arbitrary graph node values"} ::node [:set ::node]])++(mr/def ::stop+ [:= ::stop])(mu/defn transitive :- [:sequential ::node]"Given a graph and `key-seq`, returns a seq of all transitive children of those starting keys.@@ -138,7 +145,7 @@If `f` ever returns `:metabase.graph.core/stop`, `keep-children` does not include that in the results and does notrecurse down the current node's children."- [f :- [:-> ::node :any]+ [f :- [:-> ::node [:maybe [:or ::stop ::node]]]children :- ::child-map](let [all-nodes (all-map-nodes children)full-parent-map (or (->> childrendiff --git a/src/metabase/indexed_entities/models/model_index.clj b/src/metabase/indexed_entities/models/model_index.cljindex 5f63f94085c..d06bdebd199 100644--- a/src/metabase/indexed_entities/models/model_index.clj+++ b/src/metabase/indexed_entities/models/model_index.clj@@ -4,12 +4,12 @@[clojure.string :as str][clojurewerkz.quartzite.triggers :as triggers][metabase.indexed-entities.db :as indexed-entities.db]+ [metabase.indexed-entities.schema :as indexed-entities.schema];; legacy usage, do not use this in new code^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.normalize :as mbql.normalize];; model-index pk/value refs are stored as legacy field refs; validated against the legacy schema^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.schema :as mbql.s][metabase.lib.schema.common :as lib.schema.common]- [metabase.lib.schema.id :as lib.schema.id][metabase.models.interface :as mi][metabase.permissions.core :as perms][metabase.query-processor.core :as qp]@@ -19,7 +19,6 @@[metabase.util.cron :as u.cron][metabase.util.log :as log][metabase.util.malli :as mu]- [metabase.util.malli.registry :as mr][methodical.core :as methodical][toucan2.core :as t2]))@@ -87,16 +86,10 @@{:field-ref field-ref:valid-clauses [:field :expression]}))))-(mr/def ::model-index- [:map- [:model_id ::lib.schema.id/card]- [:value_ref some?]- [:pk_ref some?]])-(mu/defn ^:private fetch-values- [model-index :- ::model-index]+ [model-index :- ::indexed-entities.schema/model-index](let [model (indexed-entities.db/card (:model_id model-index))- fix (mu/fn [field-ref :- some?+ fix (mu/fn [field-ref :- ::mbql.s/FieldOrExpressionRefbase-type :- ::lib.schema.common/base-type];; stored value/pk refs are legacy MBQL; normalize as legacy before use(-> field-ref #_{:clj-kondo/ignore [:deprecated-var]} mbql.normalize/normalize-field-ref (fix-expression-refs base-type)))@@ -132,10 +125,7 @@(mu/defn add-values!"Add indexed values to the model_index_value table."- [model-index :- [:merge- ::model-index- [:map- [:id pos-int?]]]]+ [model-index :- ::indexed-entities.schema/model-index](let [[error-message values-to-index] (fetch-values model-index)current-index-values (into #{}(map (juxt :model_pk :name))diff --git a/src/metabase/indexed_entities/schema.clj b/src/metabase/indexed_entities/schema.cljindex 23162d4336d..989b63579f0 100644--- a/src/metabase/indexed_entities/schema.clj+++ b/src/metabase/indexed_entities/schema.clj@@ -15,17 +15,10 @@(mr/def ::model-index"A ModelIndex as selected from the app DB: every column of `:model_index`."- [:map {:closed true}- [:id ms/PositiveInt]- [:model_id [:maybe :int]]- [:pk_ref ::model-index.pk-ref]- [:value_ref ::model-index.value-ref]- [:schedule :string]- [:state :string]- [:indexed_at [:maybe ms/TemporalInstant]]- [:error [:maybe :string]]- [:created_at ms/TemporalInstant]- [:creator_id ::lib.schema.id/user]])+ [:merge+ ::model-index.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::model-index.update"What an update (or insert) of a ModelIndex accepts: every column of `:model_index` except `id`, all optional."@@ -42,10 +35,9 @@(mr/def ::model-index-value"A ModelIndexValue as selected from the app DB: every column of `:model_index_value`."- [:map {:closed true}- [:model_index_id [:maybe ms/PositiveInt]]- [:model_pk :int]- [:name :string]])+ [:merge+ ::model-index-value.update+ [:map {:closed true}]])(mr/def ::model-index-value.update"What an update (or insert) of a ModelIndexValue accepts: every column of `:model_index_value` except `id`, all optional."diff --git a/src/metabase/indexes/reconcile.clj b/src/metabase/indexes/reconcile.cljindex 22d856eb66e..ade4e07db06 100644--- a/src/metabase/indexes/reconcile.clj+++ b/src/metabase/indexes/reconcile.clj@@ -4,10 +4,12 @@`driver/fetch-table-indexes` to confirm it landed."(:require[metabase.driver :as driver]+ [metabase.indexes.schema :as indexes.schema][metabase.util :as u][metabase.util.log :as log][metabase.util.malli :as mu]- [metabase.util.malli.registry :as mr]))+ [metabase.util.malli.registry :as mr]+ [metabase.warehouses.schema]))(set! *warn-on-reflection* true)@@ -25,38 +27,35 @@[:style {:optional true} [:maybe :string]][:key-columns {:optional true} [:sequential [:maybe :string]]]])-(mu/defn match-key :- ::match-key- "The [[::match-key]] for a warehouse index map (see the schema for how each kind is keyed)."- [{:keys [kind key-columns] am :access-method nm :name} :- [:map- [:kind :keyword]- [:key-columns [:sequential [:maybe :string]]]- [:access-method {:optional true} [:maybe :string]]- [:name {:optional true} [:maybe :string]]]]+(mu/defn- ->match-key :- ::match-key+ "The [[::match-key]] for an index of `kind` named `nm` with access method `am` over `key-columns`."+ [kind :- :keyword+ nm :- [:maybe :string]+ am :- [:maybe :string]+ key-columns :- [:sequential [:maybe :string]]](cond(= kind :distkey) {:kind :distkey :style am :key-columns key-columns}(contains? unnamed-inline-kinds kind) {:kind kind :key-columns key-columns}:else {:kind :named :name nm}))+(mu/defn match-key :- ::match-key+ "The [[::match-key]] for a warehouse index map (see the schema for how each kind is keyed)."+ [{:keys [kind key-columns] am :access-method nm :name} :- ::driver/table-index]+ (->match-key kind nm am key-columns))+(mu/defn index-name :- :string"Physical index name for a structured def: a named kind's `:name`, else its `:kind` as a string (one inline key pertransform)."- [structured :- [:map- [:name {:optional true} [:maybe :string]]- [:kind [:or :keyword :string]]]]+ [structured :- ::indexes.schema/index-structured](or (:name structured) (name (:kind structured))))(mu/defn managed-match-key :- ::match-key"The [[match-key]] for an index request, from its stored structured definition and index name."- [{:keys [index_name structured]} :- [:map- [:index_name [:maybe :string]]- [:structured :map]]]+ [{:keys [index_name structured]} :- ::indexes.schema/table-index](let [{:keys [kind style columns]} structured;; only a :key distkey has a meaningful column; :all/:even ignore any stray column the form sentkey-columns (if (and (= kind :distkey) (not= style :key)) [] (mapv :name columns))]- (match-key {:kind kind- :name index_name- :access-method (some-> style name)- :key-columns key-columns})))+ (->match-key kind index_name (some-> style name) key-columns)))(defn warehouse-key-set"Set of [[match-key]]s for the warehouse indexes, to test managed requests against with [[present-in-warehouse?]]."@@ -119,8 +118,8 @@(mu/defn merge-indexes :- [:sequential :map]"Reality-first merged index list: every warehouse index, flagged `:metabase_managed` with its `:request` when aTableIndex `row` matches ([[match-key]]), plus any request not yet present, projected from its `:structured`."- [rows :- [:sequential :map]- warehouse-maps :- [:sequential :map]]+ [rows :- [:sequential ::indexes.schema/table-index]+ warehouse-maps :- [:sequential ::driver/table-index]](let [by-key (u/index-by managed-match-key rows)present-keys (warehouse-key-set warehouse-maps)present (for [wh warehouse-maps@@ -141,7 +140,7 @@"Physical indexes on `table-name` (`schema`) in `database` via `driver/fetch-table-indexes`.Returns `nil` if the driver can't introspect indexes or the warehouse is unreachable, so callers can distinguishfetch failure from a successful empty index list."- [database :- :map+ [database :- :metabase.warehouses.schema/databaseschema :- [:maybe :string]table-name :- :string](trydiff --git a/src/metabase/indexes/schema.clj b/src/metabase/indexes/schema.cljindex f9c7adfa3bf..88595c2c59c 100644--- a/src/metabase/indexes/schema.clj+++ b/src/metabase/indexes/schema.clj@@ -104,21 +104,14 @@(mr/def ::table-index.structured"The `:structured` column of a TableIndex, decoded."- :map)+ ::index-structured)(mr/def ::table-index"A TableIndex as selected from the app DB: every column of `:metabase_table_indexes`."- [:map {:closed true}- [:id ms/PositiveInt]- [:transform_id ::lib.schema.id/transform]- [:index_name :string]- [:structured ::table-index.structured]- [:status [:or :keyword :string]]- [:error_message [:maybe :string]]- [:created_by [:maybe :int]]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]- [:last_executed_at [:maybe ms/TemporalInstant]]])+ [:merge+ ::table-index.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::table-index.update"What an update (or insert) of a TableIndex accepts: every column of `:metabase_table_indexes` except `id`, all optional."diff --git a/src/metabase/interestingness/chart/outliers.clj b/src/metabase/interestingness/chart/outliers.cljindex cdd3131b0af..682c1f2bdea 100644--- a/src/metabase/interestingness/chart/outliers.clj+++ b/src/metabase/interestingness/chart/outliers.clj@@ -57,7 +57,7 @@:value - the numeric value:modified-z-score - the modified Z-score"[values :- [:sequential number?]- labels :- [:sequential :any]]+ labels :- [:sequential ::stats.types/axis-value]](when-let [z-scores (compute-modified-z-scores values)](let [indices (vec (argops/argfilter #(> (Math/abs (double %)) modified-z-threshold) z-scores))values-vec (vec values)@@ -88,7 +88,7 @@:diff - the period-over-period change that was flagged:modified-z-score - the modified Z-score of the diff"[values :- [:sequential number?]- labels :- [:sequential :any]]+ labels :- [:sequential ::stats.types/axis-value]](let [values-vec (vec values)labels-vec (vec labels)diffs (mapv - (rest values-vec) values-vec)]diff --git a/src/metabase/interestingness/chart/types.clj b/src/metabase/interestingness/chart/types.cljindex d0d94528761..1d10ab741e9 100644--- a/src/metabase/interestingness/chart/types.clj+++ b/src/metabase/interestingness/chart/types.clj@@ -1,26 +1,32 @@(ns metabase.interestingness.chart.types"Malli schemas for chart statistics computation."(:require- [metabase.util.malli.registry :as mr]))+ [metabase.lib-be.schema :as lib-be.schema]+ [metabase.util.malli.registry :as mr]+ [metabase.util.malli.schema :as ms]));;; -------------------------------------------------- Input Schemas -------------------------------------------------+(mr/def ::axis-value+ "A value along a chart axis: the category name, x/y coordinate, or date/time label for a data point."+ [:or :string number? :boolean ms/TemporalInstant])+(mr/def ::column-type"Type of data in a column."[:enum "string" "number" "datetime" "date" "time" "boolean"])(mr/def ::column-metadata"Metadata about a column in the chart data."- [:map+ [:map {:closed true}[:name :string][:type ::column-type]])(mr/def ::series-config"Configuration for a single series in a chart."- [:map+ [:map {:closed true}[:x ::column-metadata][:y ::column-metadata]- [:x_values [:sequential :any]]+ [:x_values [:sequential ::axis-value]][:y_values [:sequential number?]][:display_name :string][:chart_type {:optional true} [:maybe :string]]@@ -28,7 +34,7 @@(mr/def ::timeline-event"A timeline event that may be relevant to the chart."- [:map+ [:map {:closed true}[:name :string][:timestamp :string][:description {:optional true} [:maybe :string]]@@ -36,10 +42,10 @@(mr/def ::chart-config"Full chart configuration received from ai-service."- [:map+ [:map {:closed true}[:series [:map-of :string ::series-config]][:timeline_events {:optional true} [:maybe [:sequential ::timeline-event]]]- [:query {:optional true} [:maybe :map]]+ [:query {:optional true} [:maybe ::lib-be.schema/maybe-legacy-query]][:display_type {:optional true} [:maybe :string]][:title {:optional true} [:maybe :string]]])@@ -47,7 +53,7 @@(mr/def ::series-summary"Basic statistical summary of a series."- [:map+ [:map {:closed true}[:min number?][:max number?][:mean number?]@@ -57,9 +63,9 @@(mr/def ::time-range"Time range covered by the chart data."- [:map- [:start :any]- [:end :any]+ [:map {:closed true}+ [:start ::axis-value]+ [:end ::axis-value][:span-description :string]])(mr/def ::trend-direction@@ -68,7 +74,7 @@(mr/def ::trend-summary"Summary of trend in time series data."- [:map+ [:map {:closed true}[:direction ::trend-direction][:overall-change-pct number?][:start-value number?]@@ -80,16 +86,16 @@(mr/def ::volatility"Volatility metrics for time series data."- [:map+ [:map {:closed true}[:level ::volatility-level][:coefficient-of-variation number?][:max-period-change-pct number?]])(mr/def ::significant-change"A significant change detected in the data."- [:map- [:from-date :any]- [:to-date :any]+ [:map {:closed true}+ [:from-date ::axis-value]+ [:to-date ::axis-value][:from-value number?][:to-value number?][:change-abs number?]@@ -101,11 +107,11 @@(mr/def ::pattern-insight"A pattern detected in the data."- [:map+ [:map {:closed true}[:type ::pattern-type][:description :string]- [:from-date {:optional true} [:maybe :any]]- [:to-date {:optional true} [:maybe :any]]])+ [:from-date {:optional true} [:maybe ::axis-value]]+ [:to-date {:optional true} [:maybe ::axis-value]]])(mr/def ::correlation-strength"Strength of correlation between series."@@ -117,26 +123,27 @@(mr/def ::correlation"Correlation between two series."- [:map+ [:map {:closed true}[:series-a :string][:series-b :string][:coefficient number?][:strength ::correlation-strength]- [:direction ::correlation-direction]])+ [:direction ::correlation-direction]+ [:aligned-sample-size :int]])(mr/def ::outlier"An outlier detected in the data."- [:map+ [:map {:closed true}[:index :int]- [:label :any]+ [:label ::axis-value][:value number?][:modified-z-score number?]])(mr/def ::cumulative-outlier"An outlier detected in cumulative data (via period-over-period diffs)."- [:map+ [:map {:closed true}[:index :int]- [:label :any]+ [:label ::axis-value][:value number?][:diff number?][:modified-z-score number?]])@@ -145,108 +152,127 @@(mr/def ::options"Options map for chart statistics computation."- [:map+ [:map {:closed true}[:deep? {:optional true} [:maybe :boolean]][:max-correlation-series {:optional true} [:maybe :int]]])+(mr/def ::stats-limits+ "Notes about data-volume limits applied before computing chart statistics."+ [:map {:closed true}+ [:downsampled-series {:optional true} [:map-of :string [:map {:closed true}+ [:original-count :int]+ [:sampled-count :int]]]]+ [:correlations-capped {:optional true} [:map {:closed true}+ [:total-series :int]+ [:max-correlated :int]]]])+;;; ---------------------------------------------- Chart Type Stats --------------------------------------------------(mr/def ::extremum"An extreme point (peak or trough) in a series, paired with its x-coordinate."- [:map- [:x :any]+ [:map {:closed true}+ [:x ::axis-value][:y number?]])(mr/def ::time-series-series-stats"Statistics for a single time series."- [:map+ [:map {:closed true}[:summary ::series-summary][:time-range ::time-range][:data-points :int][:trend ::trend-summary][:is-cumulative :boolean]- [:outliers {:optional true} [:maybe [:sequential ::outlier]]]+ [:outliers {:optional true} [:maybe [:sequential [:or ::outlier ::cumulative-outlier]]]][:volatility {:optional true} [:maybe ::volatility]][:patterns {:optional true} [:maybe [:sequential ::pattern-insight]]][:significant-changes {:optional true} [:maybe [:sequential ::significant-change]]][:most-recent-change {:optional true} [:maybe ::significant-change]]+ [:x-name {:optional true} [:maybe :string]]+ [:y-name {:optional true} [:maybe :string]][:peak {:optional true} [:maybe ::extremum]][:trough {:optional true} [:maybe ::extremum]][:above-mean {:optional true} [:maybe :int]]])(mr/def ::time-series-stats"Statistics for time series charts."- [:map+ [:map {:closed true}[:chart-type [:= :time-series]][:series-count :int][:series [:map-of :string ::time-series-series-stats]]- [:correlations {:optional true} [:maybe [:sequential ::correlation]]]])+ [:correlations {:optional true} [:maybe [:sequential ::correlation]]]+ [:limits {:optional true} ::stats-limits]])(mr/def ::category-stat"Statistics for a single category."- [:map+ [:map {:closed true}[:name :string][:value number?][:percentage {:optional true} number?]])(mr/def ::categorical-series-stats"Statistics for a single categorical series."- [:map+ [:map {:closed true}[:summary [:maybe ::series-summary]][:data-points :int][:category-count :int][:top-categories [:sequential ::category-stat]][:bottom-categories {:optional true} [:maybe [:sequential ::category-stat]]]- [:outliers {:optional true} [:maybe [:sequential ::outlier]]]])+ [:outliers {:optional true} [:maybe [:sequential ::outlier]]]+ [:x-name {:optional true} [:maybe :string]]+ [:y-name {:optional true} [:maybe :string]]])(mr/def ::categorical-stats"Statistics for categorical charts (bar, pie, etc.)."- [:map+ [:map {:closed true}[:chart-type [:= :categorical]][:series-count :int][:series [:map-of :string ::categorical-series-stats]]- [:correlations {:optional true} [:maybe [:sequential ::correlation]]]])+ [:correlations {:optional true} [:maybe [:sequential ::correlation]]]+ [:limits {:optional true} ::stats-limits]])(mr/def ::regression-stats"Linear regression statistics."- [:map+ [:map {:closed true}[:slope number?][:intercept number?][:r-squared number?]])(mr/def ::scatter-series-stats"Statistics for a single scatter series."- [:map+ [:map {:closed true}[:x-summary [:maybe ::series-summary]][:y-summary [:maybe ::series-summary]][:data-points :int]- [:sampled-points {:optional true} [:maybe [:sequential [:sequential :any]]]]- [:correlation {:optional true} [:maybe [:map+ [:sampled-points {:optional true} [:maybe [:sequential [:sequential number?]]]]+ [:correlation {:optional true} [:maybe [:map {:closed true}[:coefficient number?][:strength ::correlation-strength][:direction ::correlation-direction]]]][:regression {:optional true} [:maybe ::regression-stats]]- [:outliers {:optional true} [:maybe [:sequential ::outlier]]]])+ [:outliers {:optional true} [:maybe [:sequential ::outlier]]]+ [:x-name {:optional true} [:maybe :string]]+ [:y-name {:optional true} [:maybe :string]]])(mr/def ::scatter-stats"Statistics for scatter plots."- [:map+ [:map {:closed true}[:chart-type [:= :scatter]][:series-count :int]- [:series [:map-of :string ::scatter-series-stats]]])+ [:series [:map-of :string ::scatter-series-stats]]+ [:limits {:optional true} ::stats-limits]])(mr/def ::histogram-summary"Weighted summary statistics estimated from binned histogram data."- [:map+ [:map {:closed true}[:weighted-mean number?][:weighted-std-dev number?][:data-range number?]])(mr/def ::estimated-distribution-stats"Distribution statistics estimated from binned histogram data using weighted approximations."- [:map+ [:map {:closed true}[:estimated-percentiles [:map-of :int number?]]- [:estimated-quartiles [:map+ [:estimated-quartiles [:map {:closed true}[:q1 number?][:median number?][:q3 number?]@@ -256,8 +282,8 @@(mr/def ::histogram-structure"Structural properties of histogram bin distribution."- [:map- [:mode-bin [:maybe [:sequential :any]]]+ [:map {:closed true}+ [:mode-bin [:maybe [:tuple number? number?]]][:peak-count :int][:concentration-top3 number?][:gap-count :int]@@ -266,42 +292,46 @@(mr/def ::histogram-series-stats"Statistics for a single histogram series."- [:map+ [:map {:closed true}[:estimated-summary ::histogram-summary][:total-count :int][:data-points :int]- [:bin-data [:sequential [:sequential :any]]]+ [:bin-data [:sequential [:sequential number?]]][:distribution ::estimated-distribution-stats]- [:structure ::histogram-structure]])+ [:structure ::histogram-structure]+ [:x-name {:optional true} [:maybe :string]]+ [:y-name {:optional true} [:maybe :string]]])(mr/def ::histogram-stats"Statistics for histogram charts."- [:map+ [:map {:closed true}[:chart-type [:= :histogram]][:series-count :int]- [:series [:map-of :string ::histogram-series-stats]]])+ [:series [:map-of :string ::histogram-series-stats]]+ [:limits {:optional true} ::stats-limits]])(mr/def ::unknown-stats"Fallback stats for chart types that don't have dedicated analysis (e.g. scalar)."- [:map+ [:map {:closed true}[:chart-type [:= :unknown]][:series-count :int]- [:message :string]])+ [:message :string]+ [:limits {:optional true} ::stats-limits]])(mr/def ::chart-stats- "Union of all chart statistics types."- [:or- ::time-series-stats- ::categorical-stats- ::scatter-stats- ::histogram-stats- ::unknown-stats])+ "Union of all chart statistics types, dispatched on `:chart-type`."+ [:multi {:dispatch :chart-type}+ [:time-series ::time-series-stats]+ [:categorical ::categorical-stats]+ [:scatter ::scatter-stats]+ [:histogram ::histogram-stats]+ [:unknown ::unknown-stats]]);;; ------------------------------------------ Representation Schema ------------------------------------------------(mr/def ::generate-repr-context"Context map for generating chart statistics representation."- [:map+ [:map {:closed true}[:stats ::chart-stats][:title {:optional true} [:maybe :string]][:display-type {:optional true} [:maybe :string]]diff --git a/src/metabase/interestingness/chart/util.clj b/src/metabase/interestingness/chart/util.cljindex 80c16de3097..4c331e0a170 100644--- a/src/metabase/interestingness/chart/util.clj+++ b/src/metabase/interestingness/chart/util.clj@@ -118,7 +118,13 @@(mu/defn make-chart-result :- ::stats.types/chart-stats"Build the standard chart stats result map."- [chart-type series-data series-stats correlations]+ [chart-type :- [:enum :time-series :categorical :scatter :histogram :unknown]+ series-data :- [:map-of :string ::stats.types/series-config]+ series-stats :- [:map-of :string [:or ::stats.types/time-series-series-stats+ ::stats.types/categorical-series-stats+ ::stats.types/scatter-series-stats+ ::stats.types/histogram-series-stats]]+ correlations :- [:maybe [:sequential ::stats.types/correlation]]](cond-> {:chart-type chart-type:series-count (count series-data):series series-stats}diff --git a/src/metabase/legacy_mbql/normalize.cljc b/src/metabase/legacy_mbql/normalize.cljcindex c1433783b75..e060f2d18f9 100644--- a/src/metabase/legacy_mbql/normalize.cljc+++ b/src/metabase/legacy_mbql/normalize.cljc@@ -32,7 +32,7 @@(mu/defn normalize-or-throw :- ::mbql.s/Query"Like [[normalize]], but checks the result against the Malli schema for a legacy query, which will cause it to throwif it fails (at least in dev)."- [query :- :map]+ [query :- :metabase.lib.util/query-like](normalize query))(defn normalize-field-refdiff --git a/src/metabase/legacy_mbql/schema.cljc b/src/metabase/legacy_mbql/schema.cljcindex c6b95a3bb40..b170690793e 100644--- a/src/metabase/legacy_mbql/schema.cljc+++ b/src/metabase/legacy_mbql/schema.cljc@@ -254,14 +254,16 @@(mr/def ::ValueTypeInfo[:map- {:decode/normalize (fn [m]- (when (map? m)- (update-keys m (comp keyword u/->snake_case_en))))+ {:closed true, :decode/normalize (fn [m]+ (when (map? m)+ (update-keys m (comp keyword u/->snake_case_en)))):description (str "Type info about a value in a `:value` clause. Added automatically by `wrap-value-literals`"" middleware to values in filter clauses based on the Field in the clause.")}[:database_type {:optional true} [:maybe ::lib.schema.common/non-blank-string]][:base_type {:optional true} [:maybe ::lib.schema.common/base-type]][:semantic_type {:optional true} [:maybe ::lib.schema.common/semantic-or-relation-type]]+ [:effective_type {:optional true} [:maybe ::lib.schema.common/base-type]]+ [:coercion_strategy {:optional true} [:maybe ::lib.schema.common/coercion-strategy]][:unit {:optional true} [:maybe ::DateTimeUnit]][:name {:optional true} [:maybe ::lib.schema.common/non-blank-string]]])@@ -824,7 +826,7 @@lib.schema.expression.temporal/datetime-modes))(mr/def ::DatetimeOptions- [:map {:decode/normalize lib.schema.common/normalize-map}+ [:map {:closed true, :decode/normalize lib.schema.common/normalize-map}[:mode {:optional true} [:ref ::DatetimeOptionsMode]]])(defclause datetime@@ -1080,7 +1082,7 @@(mr/def ::StringFilterOptions[:map- {:decode/normalize lib.schema.common/normalize-map}+ {:closed true, :decode/normalize lib.schema.common/normalize-map};; default true[:case-sensitive {:optional true} :boolean]])@@ -1103,7 +1105,7 @@(mr/def ::TimeIntervalOptions[:map- {:decode/normalize lib.schema.common/normalize-map}+ {:closed true, :decode/normalize lib.schema.common/normalize-map};; Should we include partial results for the current day/month/etc? Defaults to `false`; set this to `true` to;; include them.[:include-current {:optional true} :boolean]])@@ -1207,7 +1209,7 @@(mr/def ::CaseOptions[:map- {:decode/normalize lib.schema.common/normalize-map+ {:closed true, :decode/normalize lib.schema.common/normalize-map:error/message ":case options"}[:default {:optional true} [:ref ::ExpressionArg]]])@@ -1370,7 +1372,7 @@(mr/def ::AggregationOptionsOptions"Additional options for any aggregation clause when wrapping it in `:aggregation-options`."[:map- {:error/message ":aggregation-options options"+ {:closed true, :error/message ":aggregation-options options":decode/normalize (fn [m](let [m (if (nil? m){}@@ -1379,7 +1381,9 @@;; name to use for this aggregation in the native query instead of the default name (e.g. `count`)[:name {:optional true} ::lib.schema.common/non-blank-string];; user-facing display name for this aggregation instead of the default one- [:display-name {:optional true} ::lib.schema.common/non-blank-string]])+ [:display-name {:optional true} ::lib.schema.common/non-blank-string]+ [:metabase.query-processor.util.add-alias-info/source-alias {:optional true} [:maybe :string]]+ [:metabase.query-processor.util.add-alias-info/desired-alias {:optional true} [:maybe :string]]])(defclause* aggregation-options[:and@@ -1510,7 +1514,8 @@[:display-name ::lib.schema.common/non-blank-string];; TODO -- `:id` is actually 100% required but we have a lot of tests that don't specify it because this constraint;; wasn't previously enforced; we need to go in and fix those tests and make this non-optional- [:id {:optional true} [:ref ::lib.schema.template-tag/id]]])+ [:id {:optional true} [:ref ::lib.schema.template-tag/id]]+ [:sectionid {:optional true} ::lib.schema.common/non-blank-string]]);; Example:;;@@ -1522,14 +1527,16 @@;; :snippet-id 1}(mr/def ::TemplateTag.Snippet"Schema for a native query snippet template tag."- [:merge- ::TemplateTag.Common- [:map- [:type [:= {:decode/normalize helpers/normalize-keyword} :snippet]]- [:snippet-name ::lib.schema.common/non-blank-string]- [:snippet-id ::lib.schema.id/snippet]- ;; database to which this Snippet belongs. Doesn't always seen to be specified.- [:database {:optional true} ::lib.schema.id/database]]])+ [:and+ [:merge+ ::TemplateTag.Common+ [:map {:closed true}+ [:type [:= {:decode/normalize helpers/normalize-keyword} :snippet]]+ [:snippet-name ::lib.schema.common/non-blank-string]+ [:snippet-id ::lib.schema.id/snippet]+ ;; database to which this Snippet belongs. Doesn't always seen to be specified.+ [:database {:optional true} ::lib.schema.id/database]]]+ [:ref ::lib.schema.template-tag/disallow-dimension]]);; Example:;;@@ -1540,15 +1547,19 @@;; :card-id 1635}(mr/def ::TemplateTag.SourceQuery"Schema for a source query template tag."- [:merge- ::TemplateTag.Common- [:map- [:type [:= {:decode/normalize helpers/normalize-keyword} :card]]- [:card-id ::lib.schema.id/card]]])+ [:and+ [:merge+ ::TemplateTag.Common+ [:map {:closed true}+ [:type [:= {:decode/normalize helpers/normalize-keyword} :card]]+ [:card-id ::lib.schema.id/card]+ [:default {:optional true} [:ref ::lib.schema.parameter/parameter.value]]+ [:required {:optional true} :boolean]]]+ [:ref ::lib.schema.template-tag/disallow-dimension]])(mr/def ::TemplateTag.SourceFilter"Schema for a single source-filter applied to a table template tag."- [:map+ [:map {:closed true}[:field-id ::lib.schema.id/field][:op (into [:enum] lib.schema.template-tag/allowed-source-filter-ops)][:value [:ref ::lib.schema.parameter/parameter.value]]])@@ -1563,13 +1574,17 @@;; :source-filters [{:op :> :field-id 3 :value 500}]}(mr/def ::TemplateTag.SourceTable"Schema for a source query template tag."- [:merge- ::TemplateTag.Common- [:map- [:type [:= {:decode/normalize helpers/normalize-keyword} :table]]- [:table-id ::lib.schema.id/table]- [:emit-alias {:optional true} :boolean]- [:source-filters {:optional true} [:sequential [:ref ::TemplateTag.SourceFilter]]]]])+ [:and+ [:merge+ ::TemplateTag.Common+ [:map {:closed true}+ [:type [:= {:decode/normalize helpers/normalize-keyword} :table]]+ [:table-id ::lib.schema.id/table]+ [:emit-alias {:optional true} :boolean]+ [:source-filters {:optional true} [:sequential [:ref ::TemplateTag.SourceFilter]]]+ [:default {:optional true} [:ref ::lib.schema.parameter/parameter.value]]+ [:required {:optional true} :boolean]]]+ [:ref ::lib.schema.template-tag/disallow-dimension]])(mr/def ::TemplateTag.Value.Common"Stuff shared between the Field filter and raw value template tag schemas."@@ -1594,9 +1609,9 @@`:metabase.lib.schema.template-tag/field-filter.options`; the map stays open there and here because these optionsare merged into the parameter value the QP builds for the tag."[:map- {:decode/normalize (fn [m]- (when (map? m)- (update-keys m lib.schema.common/normalize-keyword)))}+ {:closed true, :decode/normalize (fn [m]+ (when (map? m)+ (update-keys m lib.schema.common/normalize-keyword)))}[:case-sensitive {:optional true} :boolean][:include-current {:optional true} :boolean]])@@ -1604,7 +1619,7 @@"Schema for a field filter template tag."[:merge::TemplateTag.Value.Common- [:map+ [:map {:closed true}[:type [:= {:decode/normalize helpers/normalize-keyword} :dimension]][:dimension [:ref ::field]][:alias {:optional true} :string]@@ -1630,7 +1645,7 @@"Schema for a temporal unit template tag."[:merge::TemplateTag.Value.Common- [:map+ [:map {:closed true}[:type [:= {:decode/normalize helpers/normalize-keyword} :temporal-unit]][:dimension [:ref ::field]][:alias {:optional true} :string]]])@@ -1645,15 +1660,18 @@;; :default "1"}(mr/def ::TemplateTag.RawValue"Schema for a raw value template tag."- [:merge- ::TemplateTag.Value.Common- [:map- [:type- [:ref- {:description- "`:type` is used be the FE to determine which type of widget to display for the template tag, and to determine- which types of parameters are allowed to be passed in for this template tag."}]- ::lib.schema.template-tag/raw-value.type]]])+ [:and+ [:merge+ ::TemplateTag.Value.Common+ [:map {:closed true}+ [:type+ [:ref+ {:description+ "`:type` is used be the FE to determine which type of widget to display for the template tag, and to determine+ which types of parameters are allowed to be passed in for this template tag."}+ ::lib.schema.template-tag/raw-value.type]]+ [:widget-type {:optional true} [:maybe [:ref ::WidgetType]]]]]+ [:ref ::lib.schema.template-tag/disallow-dimension]])(mr/def ::TemplateTag"Schema for a template tag as specified in a native query. There are four types of template tags, differentiated by@@ -1726,12 +1744,40 @@(remove-empty-keys m {:non-empty-keys #{:template-tags}:non-nil-keys #{:collection}}))))+(def ^:private inner-query-internal-keys+ "Map entries for the keys the query processor adds to an MBQL 5 stage, which conversion to legacy MBQL copies onto the inner query."+ [[:parameters {:optional true} [:maybe [:ref ::lib.schema.parameter/parameters]]]+ [:qp/stage-is-from-source-card {:optional true} [:ref ::lib.schema.id/card]]+ [:qp/stage-had-source-card {:optional true} [:ref ::lib.schema.id/card]]+ [:qp/skip-persisted-cache {:optional true} :boolean]+ [:persisted-info/native {:optional true} ::lib.schema.common/non-blank-string]+ [:source-query/model? {:optional true} :boolean]+ [:source-query/native-model? {:optional true} [:maybe :boolean]]+ [:metabase.query-processor.middleware.add-remaps/remaps {:optional true} [:ref :metabase.lib.schema/external-remappings]]+ [:metabase.query-processor.middleware.cumulative-aggregations/replaced-indexes {:optional true} [:set [:int {:min 0}]]]])++(mr/def ::native-query-form+ "The native query itself: a SQL string, or for drivers like MongoDB a query in the driver's own shape."+ [:or+ :string+ [:schema {::mr/deliberately-open true+ :description "a driver's native query when it is not a string, e.g. a MongoDB pipeline; its shape is the driver's"}+ :some]])+(mr/def ::NativeQuery.Common[:and- [:map- [:template-tags {:optional true} [:ref ::TemplateTagMap]]- ;; collection (table) this query should run against. Needed for MongoDB- [:collection {:optional true} [:maybe ::lib.schema.common/non-blank-string]]]+ (into+ [:map+ [:template-tags {:optional true} [:ref ::TemplateTagMap]]+ ;; collection (table) this query should run against. Needed for MongoDB+ [:collection {:optional true} [:maybe ::lib.schema.common/non-blank-string]]+ [:params {:optional true} [:maybe [:sequential [:ref ::lib.schema.literal/param-value]]]]+ [:projections {:optional true} [:maybe [:sequential :string]]]+ [:mbql? {:optional true} [:maybe :boolean]]+ [:qp/table-name {:optional true} [:maybe :string]]+ [:query-permissions/referenced-card-ids {:optional true} [:maybe [:set ::lib.schema.id/card]]]+ [:query-permissions/sandboxed-table {:optional true} [:ref ::lib.schema.id/table]]]+ inner-query-internal-keys)(lib.schema.common/disallowed-keys{:lib/type "Legacy MBQL inner queries must not have :lib/type":type "An inner query must not include :type, this will cause us to mix it up with an outer query"@@ -1744,8 +1790,8 @@[:merge{:decode/normalize #'remove-empty-keys-from-native-inner-query}::NativeQuery.Common- [:map- [:query :some]]]+ [:map {:closed true}+ [:query ::native-query-form]]](lib.schema.common/disallowed-keys{:native "A top-level native inner query should have the :query key, not :native"})])@@ -1754,8 +1800,8 @@[:merge{:decode/normalize #'remove-empty-keys-from-native-inner-query}::NativeQuery.Common- [:map- [:native :some]]]+ [:map {:closed true}+ [:native ::native-query-form]]](lib.schema.common/disallowed-keys{:query "A top-level native inner query should have the :native key, not :query"})])@@ -1853,6 +1899,17 @@(mr/def ::legacy-column-metadata.qualified-keys(legacy-column-metadata-qualified-keys-schema))+(mr/def ::driver-column+ "A column as a driver's `execute-reducible-query` reports it, before annotation adds the rest of its metadata."+ [:map {:closed true}+ [:name :string]+ [:base_type {:optional true} [:maybe ::lib.schema.common/base-type]]+ [:effective_type {:optional true} [:maybe ::lib.schema.common/base-type]]+ [:semantic_type {:optional true} [:maybe ::lib.schema.common/semantic-or-relation-type]]+ [:database_type {:optional true} [:maybe :string]]+ [:display_name {:optional true} [:maybe :string]]+ [:field_ref {:optional true} [:maybe [:ref ::Reference]]]])+(mr/def ::legacy-column-metadata"Schema for a single legacy metadata column. This is the pre-Lib equivalent of`:metabase.lib.schema.metadata/column`."@@ -1919,7 +1976,8 @@[:ref ::legacy-column-metadata.qualified-keys]](lib.schema.common/disallowed-keys{:lib/type "Legacy results metadata should not have :lib/type, use :metabase.lib.schema.metadata/column for Lib metadata"- :model/inner_ident ":model/inner_ident is deprecated"})+ :model/inner_ident ":model/inner_ident is deprecated"+ :lib/model-display-name ":lib/model-display-name is deprecated"})(letfn [(disallowed-key? [k](or (not (keyword? k))(let [disallowed-char (if (qualified-keyword? k)@@ -1970,7 +2028,7 @@[:field \"my_field\" {:base-type :field/Integer, :join-alias \"my_join_alias\"}]"[:and[:map- {:decode/normalize lib.schema.common/normalize-map}+ {:closed true, :decode/normalize lib.schema.common/normalize-map}[:source-table{:optional true:description "*What* to JOIN. Self-joins can be done by using the same `:source-table` as in the query where@@ -2030,7 +2088,16 @@{:optional true:description "Metadata about the source query being used, if pulled in from a Card via the`:source-table \"card__id\"` syntax. added automatically by the `resolve-card-id-source-tables` middleware."}- [:maybe [:sequential [:ref ::legacy-column-metadata]]]]]+ [:maybe [:sequential [:ref ::legacy-column-metadata]]]]+ [:qp/is-implicit-join+ {:optional true+ :description "Set by the `add-implicit-joins` middleware to mark a join it generated."}+ :boolean]+ [:fk-field-name {:optional true} [:maybe :string]]+ [:fk-join-alias {:optional true} [:maybe ::lib.schema.join/alias]]+ [:qp/keep-default-join-alias {:optional true} :boolean]+ [:qp/stage-is-from-source-card {:optional true} [:ref ::lib.schema.id/card]]+ [:parameters {:optional true} [:maybe [:ref ::lib.schema.parameter/parameters]]]];; additional constraints[:fn{:error/message "Joins must have either a `source-table` or `source-query`, but not both."}@@ -2113,24 +2180,31 @@(mr/def ::MBQLInnerQuery[:and- [:map- {:decode/normalize lib.schema.common/normalize-map}- [:source-query {:optional true} [:ref ::SourceQuery]]- [:source-table {:optional true} [:ref ::SourceTable]]- [:aggregation {:optional true} [:ref ::Aggregations]]- [:breakout {:optional true} [:ref ::Breakouts]]- [:expressions {:optional true} [:ref ::Expressions]]- [:fields {:optional true} [:ref ::Fields]]- [:filter {:optional true} [:ref ::Filter]]- [:limit {:optional true} nat-int?]- [:order-by {:optional true} [:ref ::OrderBys]]- [:page {:optional true} [:ref :metabase.lib.schema/page]]- [:joins {:optional true} [:ref ::Joins]]- [:source-metadata- {:optional true- :description "Info about the columns of the source query. Added in automatically by middleware. This metadata is+ (into+ [:map+ {:closed true, :decode/normalize lib.schema.common/normalize-map}+ [:source-query {:optional true} [:ref ::SourceQuery]]+ [:source-table {:optional true} [:ref ::SourceTable]]+ [:aggregation {:optional true} [:ref ::Aggregations]]+ [:breakout {:optional true} [:ref ::Breakouts]]+ [:expressions {:optional true} [:ref ::Expressions]]+ [:fields {:optional true} [:ref ::Fields]]+ [:filter {:optional true} [:ref ::Filter]]+ [:limit {:optional true} nat-int?]+ [:order-by {:optional true} [:ref ::OrderBys]]+ [:page {:optional true} [:ref :metabase.lib.schema/page]]+ [:joins {:optional true} [:ref ::Joins]]+ [:source-metadata+ {:optional true+ :description "Info about the columns of the source query. Added in automatically by middleware. This metadata isprimarily used to let power things like binning when used with Field Literals instead of normal Fields."}- [:maybe [:sequential [:ref ::legacy-column-metadata]]]]]+ [:maybe [:sequential [:ref ::legacy-column-metadata]]]]+ [:qp/added-implicit-fields? {:optional true} :boolean]+ [:query-permissions/sandboxed-table {:optional true} [:ref ::lib.schema.id/table]]+ [:metabase-enterprise.sandbox.query-processor.middleware.sandboxing/sandbox? {:optional true} :boolean]+ [:metabase.query-processor.middleware.add-implicit-joins/reused-join-aliases {:optional true} [:set :string]]+ [:pivot {:optional true} [:ref :metabase.lib.schema/pivot]]]+ inner-query-internal-keys);; remove empty query keys; this is done AFTER the map schema above because normalizing things like;; `::Aggregations` will remove things like the `ROWS` aggregation which was removed in MBQL 4.;; e.g. the schema above will normalize@@ -2254,7 +2328,7 @@{:decode/normalize #'normalize-query};; need to move source metadata to the correct location FIRST so it gets normalized by the schema below[:ref ::CheckQueryDoesNotHaveSourceMetadata]- [:map+ [:map {:closed true}[:database {:optional true} ::DatabaseID][:type[:enum@@ -2280,6 +2354,35 @@:description "Used when recording info about this run in the QueryExecution log; things like context query wasran in and User who ran it."}[:maybe [:ref ::lib.schema.info/info]]]+ [:cache-strategy {:optional true} [:maybe :metabase.lib.schema/cache-strategy]]+ [:impersonation/role {:optional true} ::lib.schema.common/non-blank-string]+ [:impersonation/admin? {:optional true} :boolean]+ [:impersonation/allow-write? {:optional true} :boolean]+ [:qp/compiled {:optional true} :metabase.lib.schema/compiled-native-query]+ [:qp/compiled-inline {:optional true} :metabase.lib.schema/compiled-native-query]+ [:async? {:optional true} :boolean]+ [:was-pivot {:optional true} [:maybe :boolean]]+ [:pivot-rows {:optional true} [:maybe [:sequential [:int {:min 0}]]]]+ [:pivot-cols {:optional true} [:maybe [:sequential [:int {:min 0}]]]]+ [:pivot-measures {:optional true} [:maybe [:sequential [:int {:min 0}]]]]+ [:show-row-totals {:optional true} [:maybe :boolean]]+ [:show-column-totals {:optional true} [:maybe :boolean]]+ [:viz-settings {:optional true} [:maybe [:ref ::lib.schema.common/visualization-settings]]]+ [:user-parameters {:optional true} [:maybe [:ref ::lib.schema.parameter/parameters]]]+ [:qp/source-card-id {:optional true} [:ref ::lib.schema.id/card]]+ [:qp/skip-result-metadata-persistence {:optional true} :boolean]+ [:qp.pivot/unremapped-breakout-combination {:optional true} [:sequential [:int {:min 0}]]]+ [:qp.pivot/remapped-breakout-combination {:optional true} [:maybe [:sequential [:int {:min 0}]]]]+ [:qp.pivot/num-remapped-cols {:optional true} [:int {:min 0}]]+ [:qp.pivot/num-unremapped-breakouts {:optional true} [:int {:min 0}]]+ [:qp.pivot/num-remapped-breakouts {:optional true} [:int {:min 0}]]+ [:qp.pivot/remapped-indexes {:optional true} [:map-of [:int {:min 0}] [:int {:min 0}]]]+ [:query-permissions/referenced-card-ids {:optional true} [:maybe [:set ::lib.schema.id/card]]]+ [:destination-database/id {:optional true} ::DatabaseID]+ [:metabase.query-processor.middleware.add-remaps/external-remaps {:optional true} :metabase.lib.schema/external-remappings]+ [:metabase-enterprise.sandbox.query-processor.middleware.sandboxing/original-metadata+ {:optional true}+ :metabase.lib.schema/sandboxing.original-metadata];;;; ACTIONS;;diff --git a/src/metabase/legacy_mbql/util.cljc b/src/metabase/legacy_mbql/util.cljcindex ac57f3f6ee4..bdd442fe10e 100644--- a/src/metabase/legacy_mbql/util.cljc+++ b/src/metabase/legacy_mbql/util.cljc@@ -75,7 +75,7 @@Prefer [[metabase.lib.core/primary-source-table-id]] going forward."{:arglists '([outer-query]), :deprecated "0.57.0"}- [{{source-table-id :source-table, source-query :source-query} :query, query-type :type, :as query} :- [:maybe :map]]+ [{{source-table-id :source-table, source-query :source-query} :query, query-type :type, :as query} :- [:maybe ::mbql.s/Query]](cond;; for native queries, there's no source table to resolve(not= query-type :query)@@ -104,7 +104,8 @@(mu/defn expression-with-name :- ::mbql.s/FieldOrExpressionDef"Return the expression referenced by a given `expression-name`."{:deprecated "0.57.0"}- [inner-query expression-name :- ::lib.schema.common/non-blank-string]+ [inner-query :- ::mbql.s/MBQLInnerQuery+ expression-name :- ::lib.schema.common/non-blank-string](loop [{:keys [expressions source-query]} inner-query, found #{}](when (seq expressions)(assert (every? string? (keys expressions))@@ -130,7 +131,8 @@support nested queries, you'll need to keep tract of how many `:source-query`s deep you've traveled; pass in thisnumber to as optional arg `nesting-level` to make sure you reference aggregations at the right level of nesting."{:deprecated "0.57.0"}- ([query index]+ ([query :- ::mbql.s/Query+ index :- nat-int?](aggregation-at-index query index 0))([query :- ::mbql.s/Query@@ -199,9 +201,10 @@"Like [[clojure.core/update]], but for the options in a `:field`, `:expression`, or `:aggregation` clause.DEPRECATED: Use MBQL 5 + [[metabase.lib.core/update-options]] going forward."- {:arglists '([field-or-ag-ref-or-expression-ref f & args]), :deprecated "0.57.0"}- [[clause-type id-or-name opts] :- ::mbql.s/Reference f & args]- (let [opts (not-empty (remove-empty (apply f opts args)))]+ {:arglists '([field-or-ag-ref-or-expression-ref f]), :deprecated "0.57.0"}+ [[clause-type id-or-name opts] :- ::mbql.s/Reference+ f :- ifn?]+ (let [opts (not-empty (remove-empty (f opts)))];; `:field` clauses should have a `nil` options map if there are no options. `:aggregation` and `:expression`;; should get the arg removed if it's `nil` or empty. (For now. In the future we may change this if we make the;; 3-arg versions the "official" normalized versions.)@@ -216,7 +219,7 @@DEPRECATED: Use MBQL 5 + [[metabase.lib.core/update-options]] going forward."{:deprecated "0.57.0"}[clause & kvs]- (apply update-field-options clause assoc kvs))+ (update-field-options clause #(apply assoc % kvs)))(defn with-temporal-unit"Set the `:temporal-unit` of a `:field` clause to `unit`.diff --git a/src/metabase/lib/aggregation.cljc b/src/metabase/lib/aggregation.cljcindex 09382da550a..f7e1c768ad8 100644--- a/src/metabase/lib/aggregation.cljc+++ b/src/metabase/lib/aggregation.cljc@@ -45,10 +45,15 @@(mu/defn resolve-aggregation-by-name :- ::lib.schema.aggregation/aggregation"Attempt to find an aggregation with `ag-name`. This is mostly for dealing with super broken `:field` refs inparameters that do stuff like `[:field \"count\" nil]` inside [[metabase.lib.field.resolution]]."- ([query stage-number ag-name]+ ([query :- ::lib.schema/query+ stage-number :- :int+ ag-name :- :string](resolve-aggregation-by-name query stage-number (aggregations query stage-number) ag-name))- ([query stage-number ags ag-name :- :string]+ ([query :- ::lib.schema/query+ stage-number :- :int+ ags :- [:maybe [:sequential ::lib.schema.aggregation/aggregation]]+ ag-name :- :string];; PERF: Index these by name on the stage?(m/find-first(fn [aggregation]@@ -339,7 +344,8 @@(mu/defn aggregate :- ::lib.schema/query"Adds an aggregation to query."- ([query aggregable]+ ([query :- ::lib.schema/query+ aggregable :- ::aggregable](aggregate query -1 aggregable))([query :- ::lib.schema/query@@ -352,7 +358,7 @@(mu/defn aggregations :- [:maybe [:sequential ::lib.schema.aggregation/aggregation]]"Get the aggregations in a given stage of a query."- ([query]+ ([query :- ::lib.schema/query](aggregations query -1))([query :- ::lib.schema/query@@ -361,7 +367,7 @@(mu/defn aggregations-metadata :- [:maybe [:sequential ::lib.metadata.calculation/column-metadata-with-source]]"Get metadata about the aggregations in a given stage of a query."- ([query]+ ([query :- ::lib.schema/query](aggregations-metadata query -1))([query :- ::lib.schema/query@@ -399,7 +405,7 @@(mu/defn available-aggregation-operators :- [:maybe [:sequential ::operator-with-columns]]"Returns the available aggregation operators for the stage with `stage-number` of `query`.If `stage-number` is omitted, uses the last stage."- ([query]+ ([query :- ::lib.schema/query](available-aggregation-operators query -1))([query :- ::lib.schema/query@@ -427,33 +433,33 @@(map #(assoc % :lib/type :operator/aggregation)))lib.schema.aggregation/aggregation-operators)))))+(mr/def ::selected-operator-with-columns+ [:merge+ ::lib.schema.aggregation/operator+ [:map+ [:columns {:optional true} [:sequential ::lib.schema.metadata/column]]+ [:selected? {:optional true} :boolean]]])+(mu/defn aggregation-clause :- ::lib.schema.aggregation/aggregation"Returns a standalone aggregation clause for an `aggregation-operator` anda `column`.For aggregations requiring an argument `column` is mandatory, otherwiseit is optional."- ([aggregation-operator :- ::lib.schema.aggregation/operator]+ ([aggregation-operator :- ::selected-operator-with-columns](if-not (:requires-column? aggregation-operator)(lib.options/ensure-uuid [(:short aggregation-operator) {}])(throw (ex-info (lib.util/format "aggregation operator %s requires an argument"(:short aggregation-operator)){:aggregation-operator aggregation-operator}))))- ([aggregation-operator :- ::lib.schema.aggregation/operator- column]+ ([aggregation-operator :- ::selected-operator-with-columns+ column :- :metabase.lib.common/op-arg](lib.options/ensure-uuid [(:short aggregation-operator) {} (lib.common/->op-arg column)])))-(mr/def ::selected-operator-with-columns- [:merge- ::lib.schema.aggregation/operator- [:map- [:columns {:optional true} [:sequential ::lib.schema.metadata/column]]- [:selected? {:optional true} :boolean]]])-(mu/defn selected-aggregation-operators :- [:maybe [:sequential ::selected-operator-with-columns]]"Mark the operator and the column (if any) in `agg-operators` selected by `agg-clause`."[agg-operators :- [:maybe [:sequential ::operator-with-columns]]- agg-clause]+ agg-clause :- ::lib.schema.aggregation/aggregation](when (seq agg-operators)(let [[op _ agg-col] agg-clauseagg-temporal-unit (-> agg-col lib.options/options :temporal-unit)]@@ -483,7 +489,8 @@(-> (lib/query ...)(lib/aggregate (lib/avg ...))(as-> <> (lib/order-by <> (lib/aggregation-ref <> 0))))"- ([query ag-index]+ ([query :- ::lib.schema/query+ ag-index :- nat-int?](aggregation-ref query -1 ag-index))([query :- ::lib.schema/query@@ -531,7 +538,7 @@(mu/defn remove-all-aggregations :- ::lib.schema/query"Remove all aggregations from a query stage."- ([query]+ ([query :- ::lib.schema/query](remove-all-aggregations query -1))([query :- ::lib.schema/querydiff --git a/src/metabase/lib/binning.cljc b/src/metabase/lib/binning.cljcindex dcb84cf7590..09f2e901ec6 100644--- a/src/metabase/lib/binning.cljc+++ b/src/metabase/lib/binning.cljc@@ -12,6 +12,7 @@[metabase.lib.schema.binning :as lib.schema.binning][metabase.lib.schema.common :as lib.schema.common][metabase.lib.schema.metadata :as lib.schema.metadata]+ [metabase.lib.schema.ref :as lib.schema.ref][metabase.lib.util :as lib.util][metabase.util :as u][metabase.util.i18n :as i18n]@@ -45,7 +46,8 @@Pass `nil` `binning` to remove any binning."{:style/indent [:form]}- [x binning :- [:maybe [:or ::lib.schema.binning/binning.resolved ::lib.schema.binning/binning-option]]]+ [x :- [:or ::lib.schema.metadata/column ::lib.schema.ref/ref]+ binning :- [:maybe [:or ::lib.schema.binning/binning.resolved ::lib.schema.binning/binning-option]]](with-binning-method x (->binning-options (if (contains? binning :mbql)(:mbql binning)binning))))@@ -64,7 +66,7 @@"Get the current binning options associated with `x`, if any. The options come back tagged with `:lib/type` and a`:metadata-fn` closure for [[metabase.lib.metadata.calculation/display-info]]; strip those before putting them backinto a query."- [x]+ [x :- [:or ::lib.schema.metadata/column ::lib.schema.ref/ref]](binning-method x))(defmulti available-binning-strategies-method@@ -81,12 +83,13 @@(mu/defn available-binning-strategies :- [:maybe [:sequential [:ref ::lib.schema.binning/binning-option]]]"Get a set of available binning strategies for `x`. Returns nil if none are available."- ([query x]+ ([query :- ::lib.schema/query+ x :- [:or ::lib.schema.metadata/column ::lib.schema.ref/ref]](available-binning-strategies query -1 x))([query :- ::lib.schema/querystage-number :- :int- x]+ x :- [:or ::lib.schema.metadata/column ::lib.schema.ref/ref]](available-binning-strategies-method query stage-number x)))(mu/defn default-auto-bin :- ::lib.schema.binning/binning-optiondiff --git a/src/metabase/lib/binning/util.cljc b/src/metabase/lib/binning/util.cljcindex fe28aceac12..528206c555d 100644--- a/src/metabase/lib/binning/util.cljc+++ b/src/metabase/lib/binning/util.cljc@@ -146,7 +146,7 @@"Humanize binning: extend interval to start and end on a \"nice\" number and, when number of bins is fixed, have a\"nice\" step (bin width)."[strategy :- ::lib.schema.binning/strategy- {:keys [min-value max-value bin-width num-bins]} :- [:map+ {:keys [min-value max-value bin-width num-bins]} :- [:map {:closed true}[:min-value number?][:max-value number?][:bin-width {:optional true} ::lib.schema.binning/bin-width]@@ -166,7 +166,11 @@"Make the current breakout a little nicer? Not 100% sure exactly how this is used, referto [[metabase.query-processor.middleware.binning/update-binned-field]]."[strategy :- ::lib.schema.binning/strategy- opts :- :map]+ opts :- [:map {:closed true}+ [:min-value number?]+ [:max-value number?]+ [:bin-width {:optional true} ::lib.schema.binning/bin-width]+ [:num-bins {:optional true} ::lib.schema.binning/num-bins]]](let [f (partial nicer-breakout* strategy)]((fixed-point f) opts)))diff --git a/src/metabase/lib/breakout.cljc b/src/metabase/lib/breakout.cljcindex 707f33cddd2..c22a9109708 100644--- a/src/metabase/lib/breakout.cljc+++ b/src/metabase/lib/breakout.cljc@@ -28,7 +28,7 @@(mu/defn breakouts :- [:maybe [:sequential ::lib.schema.expression/expression]]"Return the current breakouts"- ([query]+ ([query :- ::lib.schema/query](breakouts query -1))([query :- ::lib.schema/querystage-number :- :int]@@ -36,7 +36,7 @@(mu/defn breakouts-metadata :- [:maybe ::lib.metadata.calculation/visible-columns]"Get metadata about the breakouts in a given stage of a `query`."- ([query]+ ([query :- ::lib.schema/query](breakouts-metadata query -1))([query :- ::lib.schema/querystage-number :- :int]@@ -47,11 +47,12 @@(mu/defn breakout :- ::lib.schema/query"Add a new breakout on an expression, presumably a Field reference. Ignores attempts to add a duplicate breakout."- ([query expr]+ ([query :- ::lib.schema/query+ expr :- [:or fn? ::lib.ref/referenceable]](breakout query -1 expr))([query :- ::lib.schema/querystage-number :- :int- expr :- some?]+ expr :- [:or fn? ::lib.ref/referenceable]](let [expr (if (fn? expr) (expr query stage-number) expr)](if (lib.schema.util/distinct-mbql-clauses? (map lib.ref/ref (cons expr (breakouts query stage-number))))(lib.util/add-summary-clause query stage-number :breakout expr)@@ -97,7 +98,9 @@(mu/defn existing-breakouts :- [:maybe [:sequential {:min 1} ::lib.schema.ref/ref]]"Returns existing breakouts (as MBQL expressions) for `column` in a stage if there are any. Returns `nil` if thereare no existing breakouts."- ([query stage-number column]+ ([query :- ::lib.schema/query+ stage-number :- :int+ column :- ::lib.schema.metadata/column](existing-breakouts query stage-number column nil))([query :- ::lib.schema/query@@ -105,7 +108,7 @@column :- ::lib.schema.metadata/column{:keys [same-binning-strategy?same-temporal-bucket?], :as _options} :- [:maybe- [:map+ [:map {:closed true}[:same-binning-strategy? {:optional true, :default false} [:maybe :boolean]][:same-temporal-bucket? {:optional true, :default false} [:maybe :boolean]]]]](not-empty@@ -130,7 +133,8 @@(mu/defn remove-existing-breakouts-for-column :- ::lib.schema/query"Remove all existing breakouts against `column` if there are any in the stage in question. Disregards temporalbucketing and binning."- ([query column]+ ([query :- ::lib.schema/query+ column :- ::lib.schema.metadata/column](remove-existing-breakouts-for-column query -1 column))([query :- ::lib.schema/query@@ -161,7 +165,7 @@(mu/defn remove-all-breakouts :- ::lib.schema/query"Remove all breakouts from a query stage."- ([query]+ ([query :- ::lib.schema/query](remove-all-breakouts query -1))([query :- ::lib.schema/querydiff --git a/src/metabase/lib/card.cljc b/src/metabase/lib/card.cljcindex 08f1557a623..1e66fe34e8c 100644--- a/src/metabase/lib/card.cljc+++ b/src/metabase/lib/card.cljc@@ -69,7 +69,7 @@* `source-metadata-col` = (possibly snake_cased) column metadata from Card `:source-metadata`* `field-metadata` = Field metadata (`:metadata/column`) from the metadata provider for the Field with ID"[metadata-providerable :- ::lib.schema.metadata/metadata-providerable- source-metadata-col :- :map+ source-metadata-col :- ::lib.schema.metadata/lib-or-legacy-columncard-id :- [:maybe ::lib.schema.id/card]field-metadata :- [:maybe ::lib.schema.metadata/column]](let [source-metadata-col (-> source-metadata-col@@ -105,26 +105,28 @@;; accordingly.(not= (:semantic-type source-metadata-col) :type/FK)(assoc :fk-target-field-id nil))]- (-> col+ (-> (lib.normalize/normalize ::lib.schema.metadata/column col)lib.field.util/update-keys-for-col-from-previous-stage(merge (when card-id{:lib/source :source/card, :lib/card-id card-id}));; :effective-type is required, but not always set, see e.g.,;; [[metabase.warehouse-schema-rest.api.table/card-result-metadata->virtual-fields]]- (u/assoc-default :effective-type (:base-type col))- ;; add original display name IF not already present AND we have a value- (->> (lib.normalize/normalize ::lib.schema.metadata/column)))))+ (as-> $col (u/assoc-default $col :effective-type (:base-type $col))))))(mu/defn ->card-metadata-columns :- [:maybe [:sequential ::lib.schema.metadata/column]]"Massage possibly-legacy Card results metadata into Lib ColumnMetadata."- ([metadata-providerable cols]+ ([metadata-providerable :- ::lib.schema.metadata/metadata-providerable+ cols :- [:maybe [:or+ [:sequential ::lib.schema.metadata/lib-or-legacy-column]+ [:map {:closed true}+ [:columns [:sequential ::lib.schema.metadata/lib-or-legacy-column]]]]]](->card-metadata-columns metadata-providerable nil cols))([metadata-providerable :- ::lib.schema.metadata/metadata-providerablecard-or-id-or-nil :- [:maybe [:or ::lib.schema.id/card ::lib.schema.metadata/card]]cols :- [:maybe [:or[:sequential ::lib.schema.metadata/lib-or-legacy-column]- [:map+ [:map {:closed true}[:columns [:sequential ::lib.schema.metadata/lib-or-legacy-column]]]]]];; Card `result-metadata` SHOULD be a sequence of column infos, but just to be safe handle a map that;; contains` :columns` as well.@@ -229,12 +231,15 @@- `:own-model-query?` — when true, aggregation columns are also merged (the model's own aggregation results shouldpreserve user-customized names). When false (default), aggregation columns are skipped because the computed name(e.g. 'Sum of Price') is better than the model's custom name when the model is used as a source for an outer query."- ([result-cols model-cols]+ ([result-cols :- [:maybe [:sequential ::lib.schema.metadata/column]]+ model-cols :- [:maybe [:sequential ::lib.schema.metadata/column]]](merge-model-metadata result-cols model-cols {}))([result-cols :- [:maybe [:sequential ::lib.schema.metadata/column]]model-cols :- [:maybe [:sequential ::lib.schema.metadata/column]]{:keys [native-model? own-model-query?]- :or {native-model? false, own-model-query? false}}]+ :or {native-model? false, own-model-query? false}} :- [:map {:closed true}+ [:native-model? {:optional true} :boolean]+ [:own-model-query? {:optional true} :boolean]]](cond(empty? model-cols)(not-empty result-cols)@@ -308,7 +313,7 @@[query :- ::lib.schema/query_stage-number :- :intcard :- ::lib.schema.metadata/card- options :- [:maybe ::lib.metadata.calculation/returned-columns.options]]+ options :- [:maybe ::lib.metadata.calculation/visible-columns.options]](lib.computed/with-cache-sticky* query[::returned-columns (:id card) (lib.metadata.calculation/cacheable-options options)](fn []diff --git a/src/metabase/lib/column_group.cljc b/src/metabase/lib/column_group.cljcindex c4ad4260991..ee47d45e017 100644--- a/src/metabase/lib/column_group.cljc+++ b/src/metabase/lib/column_group.cljc@@ -10,10 +10,12 @@[metabase.lib.metadata.calculation :as lib.metadata.calculation][metabase.lib.schema.common :as lib.schema.common][metabase.lib.schema.id :as lib.schema.id]+ [metabase.lib.schema.join :as lib.schema.join][metabase.lib.schema.metadata :as lib.schema.metadata][metabase.lib.util :as lib.util][metabase.util.i18n :as i18n][metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr][metabase.util.performance :refer [select-keys some]]))(def ^:private GroupType@@ -30,13 +32,19 @@:group-type/join.explicit 2:group-type/join.implicit 3})-(def ^:private ColumnGroup+(mr/def ::column-group"Schema for the metadata returned by [[group-columns]], and accepted by [[columns-group-columns]]."[:and- [:map- [:lib/type [:= :metadata/column-group]]- [::group-type GroupType]- [::columns [:sequential [:ref ::lib.schema.metadata/column]]]]+ [:map {:closed true}+ [:lib/type [:= :metadata/column-group]]+ [::group-type GroupType]+ [::columns [:sequential [:ref ::lib.schema.metadata/column]]]+ [:join-alias {:optional true} [:ref ::lib.schema.common/non-blank-string]]+ [:table-id {:optional true} [:ref ::lib.schema.id/table]]+ [:card-id {:optional true} [:ref ::lib.schema.id/card]]+ [:fk-field-id {:optional true} [:ref ::lib.schema.id/field]]+ [:fk-field-name {:optional true} [:maybe :string]]+ [:fk-join-alias {:optional true} [:maybe ::lib.schema.join/alias]]][:multi{:dispatch ::group-type}[:group-type/main@@ -45,18 +53,15 @@;; with [[metabase.lib.join/join-condition-rhs-columns]], the alias won't be present yet, so group things by the;; joinable -- either the Card we're joining, or the Table we're joining. See #32493[:group-type/join.explicit- [:and- [:map- [:join-alias {:optional true} [:ref ::lib.schema.common/non-blank-string]]- [:table-id {:optional true} [:ref ::lib.schema.id/table]]- [:card-id {:optional true} [:ref ::lib.schema.id/card]]]- [:fn- {:error/message ":group-type/join.explicit should only have at most one of :join-alias, :table-id, or :card-id"}- (fn [m]- (>= (count (keys (select-keys m [:join-alias :table-id :card-id]))) 1))]]]+ [:fn+ {:error/message ":group-type/join.explicit should only have at most one of :join-alias, :table-id, or :card-id"}+ (fn [m]+ (>= (count (keys (select-keys m [:join-alias :table-id :card-id]))) 1))]][:group-type/join.implicit- [:map- [:fk-field-id [:ref ::lib.schema.id/field]]]]]])+ [:fn+ {:error/message ":group-type/join.implicit requires :fk-field-id"}+ (fn [m]+ (contains? m :fk-field-id))]]]])(defmethod lib.metadata.calculation/metadata-method :metadata/column-group[_query _stage-number column-group]@@ -192,7 +197,7 @@:group-type/join.implicit [(or (:fk-join-alias column-group) "")(fk-field-names (:fk-field-id column-group) "")])))-(mu/defn group-columns :- [:sequential ColumnGroup]+(mu/defn group-columns :- [:sequential ::column-group]"Given a group of columns returned by a function like [[metabase.lib.order-by/orderable-columns]], group the columnsby Table or equivalent (e.g. Saved Question) so that they're in an appropriate shape for showing in the QueryBuilder. e.g a sequence of columns like@@ -232,7 +237,7 @@(mu/defn columns-group-columns :- [:sequential ::lib.schema.metadata/column]"Get the columns associated with a column group"- [column-group :- ColumnGroup]+ [column-group :- ::column-group](::columns column-group))(defmethod lib.metadata.calculation/display-name-method :metadata/column-groupdiff --git a/src/metabase/lib/common.cljc b/src/metabase/lib/common.cljcindex bd1e164a772..f91728b1db5 100644--- a/src/metabase/lib/common.cljc+++ b/src/metabase/lib/common.cljc@@ -1,14 +1,18 @@(ns metabase.lib.common- (:refer-clojure :exclude [mapv every? #?(:clj for)])+ (:refer-clojure :exclude [mapv every? #?@(:clj [for empty?])])(:require[metabase.lib.dispatch :as lib.dispatch][metabase.lib.hierarchy :as lib.hierarchy][metabase.lib.options :as lib.options][metabase.lib.ref :as lib.ref][metabase.lib.schema.common :as schema.common]+ [metabase.lib.schema.literal :as lib.schema.literal]+ [metabase.lib.schema.mbql-clause :as lib.schema.mbql-clause]+ [metabase.lib.schema.metadata :as lib.schema.metadata][metabase.util :as u][metabase.util.malli :as mu]- [metabase.util.performance :refer [mapv #?@(:clj [every? for])]])+ [metabase.util.malli.registry :as mr]+ [metabase.util.performance :refer [mapv #?@(:clj [every? for empty?])]])#?(:cljs (:require-macros [metabase.lib.common])))(comment lib.options/keep-me@@ -16,7 +20,7 @@(mu/defn external-op :- [:maybe ::schema.common/external-op]"Convert the internal operator `clause` to the external format."- [[operator options :as clause]]+ [[operator options :as clause] :- [:maybe ::lib.schema.mbql-clause/clause]](when clause{:lib/type :lib/external-op:operator (cond-> operator@@ -77,6 +81,45 @@(map ->op-arg)args))+(defn- regex?+ "Whether `x` is a regex pattern object."+ [x]+ #?(:clj (instance? java.util.regex.Pattern x)+ :cljs (regexp? x)))++(mr/def ::op-arg+ "A value `->op-arg` can turn into a clause argument: a literal, a metadata ref, an already-built clause, a clause+ whose arguments are still any of these, an external-op map, a regex, or a (possibly nested) sequence of these."+ [:or+ :string+ :boolean+ :keyword+ number?+ nil?+ [:fn regex?]+ ::lib.schema.literal/literal+ ::lib.schema.metadata/column+ ::lib.schema.metadata/metric+ ::lib.schema.metadata/segment+ ::lib.schema.metadata/measure+ ::schema.common/external-op+ ::lib.schema.mbql-clause/clause+ [:and vector? [:cat :keyword [:? ::schema.common/options] [:* [:schema [:ref ::op-arg]]]]]+ [:sequential [:ref ::op-arg]]])++#?(:clj+ (defn- typed-argvec+ "Add `:- ::op-arg` to each positional arg symbol in `argvec`, and `:- [:* ::op-arg]` to a trailing `& more`."+ [argvec]+ (loop [args argvec, out [], variadic? false]+ (if (empty? args)+ out+ (let [arg (first args)]+ (cond+ (= arg '&) (recur (rest args) (conj out '&) true)+ variadic? (recur (rest args) (conj out arg :- [:* ::op-arg]) false)+ :else (recur (rest args) (conj out arg :- ::op-arg) false)))))))+#?(:clj(defmacro defop"Defines a clause creating function with given args.@@ -91,5 +134,5 @@:let [arglist-expr (if (contains? (set argvec) '&)(cons `list* (remove #{'&} argvec))argvec)]]- `([~@argvec]+ `([~@(typed-argvec argvec)](defop-create ~(keyword op-name) ~arglist-expr))))))diff --git a/src/metabase/lib/convert.cljc b/src/metabase/lib/convert.cljcindex c7372553f60..61a7cdd9fa2 100644--- a/src/metabase/lib/convert.cljc+++ b/src/metabase/lib/convert.cljc@@ -16,6 +16,7 @@[metabase.lib.normalize :as lib.normalize][metabase.lib.options :as lib.options][metabase.lib.schema :as lib.schema]+ [metabase.lib.schema.common :as lib.schema.common][metabase.lib.schema.expression :as lib.schema.expression][metabase.lib.schema.ref :as lib.schema.ref][metabase.lib.util :as lib.util]@@ -494,7 +495,7 @@(mu/defn- options->legacy-MBQL :- [:maybe [:map {:min 1}]]"Convert an options map in an MBQL clause to the equivalent shape for legacy MBQL. Remove `:lib/*` keys and`:effective-type`, which is not used in options maps in legacy MBQL."- [m :- [:maybe :map]]+ [m :- [:maybe ::lib.schema.common/clause-options]](->> (cond-> m;; Following construct ensures that transformation MBQL 4 -> MBQL 5 -> MBQL 4, does not add base-type where;; those were not present originally. Base types are added in [[metabase.lib.query/add-types-to-fields]].@@ -605,12 +606,12 @@(:columns stage-metadata)))(mu/defn- chain-stages- ([m]- (chain-stages m nil))+ ([stages :- [:sequential ::lib.util/query-like]]+ (chain-stages stages nil))- ([{:keys [stages]} :- [:map [:stages [:sequential :map]]]+ ([stages :- [:sequential ::lib.util/query-like]{:keys [top-level?], :or {top-level? true}, :as _opts} :- [:maybe- [:map+ [:map {:closed true}[:top-level? [:maybe :boolean]]]]];; :source-metadata aka :lib/stage-metadata is handled differently in the two formats.;; In legacy, an inner query might have both :source-query, and :source-metadata giving the metadata for that nested@@ -701,7 +702,7 @@(when (seq (:columns metadata)){:source-metadata (stage-metadata->legacy-metadata metadata)})(let [inner-query (chain-stages- (dissoc base :fields :conditions)+ (:stages base){:top-level? false})];; if [[chain-stages]] returns any additional keys like `:filter` at the top-level then we need to wrap;; it all in `:source-query` (QUE-1566, QUE-1603)@@ -761,7 +762,7 @@(let [base (merge (disqualify (dissoc query :info))(select-keys query [:info]))parameters (:parameters base)- inner-query (chain-stages base)+ inner-query (chain-stages (:stages base))query-type (if (-> query :stages last :lib/type (= :mbql.stage/native)):native:query)]@@ -777,15 +778,21 @@;; TODO: Look into whether this function can be refactored away - it's called from several places but I (Braden) think;; legacy refs shouldn't make it out of `lib.js`.+(mr/def ::unnormalized-legacy-ref+ "A legacy MBQL reference that may not be normalized yet, e.g. with string tags from JSON, or a JS array in CLJS."+ #?(:clj ::lib.schema.common/any-clause+ :cljs [:or ::lib.schema.common/any-clause [:fn {:error/message "JS array"} array?]]))+(mu/defn legacy-ref->mbql5 :- ::lib.schema.ref/ref"Convert a legacy MBQL `:field`/`:aggregation`/`:expression` reference to MBQL 5. Normalizes the reference if needed,and handles JS -> Clj conversion as needed."- ([query legacy-ref]+ ([query :- ::lib.schema/query+ legacy-ref :- ::unnormalized-legacy-ref](legacy-ref->mbql5 query -1 legacy-ref))([query :- ::lib.schema/querystage-number :- :int- legacy-ref :- some?]+ legacy-ref :- ::unnormalized-legacy-ref](let [legacy-ref (->> #?(:clj legacy-ref :cljs (js->clj legacy-ref :keywordize-keys true));; input is a legacy ref; normalize as legacy MBQL before conversion#_{:clj-kondo/ignore [:deprecated-var]}diff --git a/src/metabase/lib/convert/metadata_to_legacy.cljc b/src/metabase/lib/convert/metadata_to_legacy.cljcindex 7857b99daf2..c5ae7233b4b 100644--- a/src/metabase/lib/convert/metadata_to_legacy.cljc+++ b/src/metabase/lib/convert/metadata_to_legacy.cljc@@ -4,6 +4,7 @@;; side-effect require: registers the legacy-column-metadata schema this ns's output is checked against^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.schema][metabase.lib.normalize :as lib.normalize]+ [metabase.lib.schema.metadata :as lib.schema.metadata][metabase.util :as u][metabase.util.malli :as mu][metabase.util.memoize :as u.memo]))@@ -30,7 +31,9 @@namespaced keywords remain in kebab case)* Remove `:lib/type`"- [col :- :map]+ [col :- [:multi {:dispatch #(if (:lib/type %) :lib :legacy)}+ [:lib ::lib.schema.metadata/column.map]+ [:legacy :metabase.legacy-mbql.schema/legacy-column-metadata]]];; Intentionally using vanilla update-keys here because m.u.perf's implementation would try to assoc snake keys onto;; SnakeHatingMap which results in an exception.#_{:clj-kondo/ignore [:discouraged-var]}diff --git a/src/metabase/lib/core.cljc b/src/metabase/lib/core.cljcindex ae5dbbff466..80f6009e5fa 100644--- a/src/metabase/lib/core.cljc+++ b/src/metabase/lib/core.cljc@@ -263,12 +263,14 @@expression parser and pretty-printer in the Notebook editor, see [[expression-clause]] and [[expression-parts]].**Code Health:** Healthy. This is a core API."- ([a-query expression-name expressionable]+ ([a-query :- ::lib.schema/query+ expression-name :- ::lib.schema.common/non-blank-string+ expressionable :- ::lib.expression/expressionable](expression a-query -1 expression-name expressionable))([a-query :- ::lib.schema/querystage-number :- [:maybe :int]expression-name :- ::lib.schema.common/non-blank-string- expressionable]+ expressionable :- ::lib.expression/expressionable](lib.expression/expression a-query stage-number expression-name expressionable))); TODO: (bshepherdson, 2026-03-03) There's a third arity in `lib.expression/expression` that supports an options map; but it's a bit "gory details" so it's omitted here. I think it's only called by QP.@@ -280,7 +282,7 @@a new expression.**Code Health:** Healthy. This is a core API."- ([a-query] (expressions a-query -1))+ ([a-query :- ::lib.schema/query] (expressions a-query -1))([a-query :- ::lib.schema/querystage-number :- [:maybe :int]](lib.expression/expressions a-query stage-number)))@@ -299,7 +301,9 @@4. All columns which are *implicitly joinable* through any FK in the above.**Code Health:** Healthy. This is a core API."- ([a-query expression-position] (expressionable-columns a-query -1 expression-position))+ ([a-query :- ::lib.schema/query+ expression-position :- [:maybe nat-int?]]+ (expressionable-columns a-query -1 expression-position))([a-query :- ::lib.schema/querystage-number :- [:maybe :int]expression-position :- [:maybe nat-int?]]@@ -311,7 +315,7 @@**Code Health:** Leak. Expression columns are properly included in e.g. [[filterable-columns]]; there should not because to care about the expression columns specifically outside of Lib."- ([a-query] (expressions-metadata a-query -1))+ ([a-query :- ::lib.schema/query] (expressions-metadata a-query -1))([a-query :- ::lib.schema/querystage-number :- [:maybe :int]](lib.expression/expressions-metadata a-query stage-number)))@@ -321,7 +325,9 @@to it.**Code Health:** Deprecated. This is a test helper that crept into the public API."- ([a-query expression-name] (expression-ref a-query -1 expression-name))+ ([a-query :- ::lib.schema/query+ expression-name :- ::lib.schema.common/non-blank-string]+ (expression-ref a-query -1 expression-name))([a-query :- ::lib.schema/querystage-number :- [:maybe :int]expression-name :- ::lib.schema.common/non-blank-string]@@ -439,7 +445,8 @@- A (v2) metric**Code Health:** Healthy. This is a core API."- ([a-query aggregable]+ ([a-query :- ::lib.schema/query+ aggregable :- ::lib.aggregation/aggregable](aggregate a-query -1 aggregable))([a-query :- ::lib.schema/querystage-number :- :int@@ -448,7 +455,7 @@(mu/defn aggregations :- [:maybe [:sequential ::lib.schema.aggregation/aggregation]]"Returns the list of aggregation definitions on the target stage of `a-query`, or nil if there are none."- ([a-query] (aggregations a-query -1))+ ([a-query :- ::lib.schema/query] (aggregations a-query -1))([a-query :- ::lib.schema/querystage-number :- :int](lib.aggregation/aggregations a-query stage-number)))@@ -476,7 +483,8 @@**Code Health:** Single use. This mainly exists for the custom editor in the UI.Prefer [[available-aggregation-operators]] and [[aggregation-operator-columns]] for more \"structured\" usages."- ([a-query aggregation-position]+ ([a-query :- ::lib.schema/query+ aggregation-position :- [:maybe nat-int?]](aggregable-columns a-query -1 aggregation-position))([a-query :- ::lib.schema/querystage-number :- :int@@ -499,7 +507,7 @@that require an input column when no suitable inputs are available.**Code Health:** Healthy. This is the correct way to determine the valid set of aggregation operators."- ([a-query] (available-aggregation-operators a-query -1))+ ([a-query :- ::lib.schema/query] (available-aggregation-operators a-query -1))([a-query :- ::lib.schema/querystage-number :- :int](lib.aggregation/available-aggregation-operators a-query stage-number)))@@ -508,7 +516,7 @@"Removes all aggregations from the target stage of `a-query`.**Code Health:** Healthy, though it's an unusual thing to do."- ([a-query] (remove-all-aggregations a-query -1))+ ([a-query :- ::lib.schema/query] (remove-all-aggregations a-query -1))([a-query :- ::lib.schema/querystage-number :- :int](lib.aggregation/remove-all-aggregations a-query stage-number)))@@ -530,7 +538,8 @@This is useful for adding an [[order-by]] clause, since [[aggregations]] returns aggregation *definitions*, but[[order-by]] needs aggregation references."- ([a-query agg-index]+ ([a-query :- ::lib.schema/query+ agg-index :- nat-int?](aggregation-ref a-query -1 agg-index))([a-query :- ::lib.schema/querystage-number :- :int@@ -588,17 +597,19 @@provided they all set different `:temporal-unit`s.**Code Health:** Healthy. This is a core API."- ([a-query expr] (breakout a-query -1 expr))+ ([a-query :- ::lib.schema/query+ expr :- [:or ::lib.schema.metadata/column ::lib.schema.ref/ref ifn?]]+ (breakout a-query -1 expr))([a-query :- ::lib.schema/querystage-number :- :int- expr :- some?]+ expr :- [:or ::lib.schema.metadata/column ::lib.schema.ref/ref ifn?]](lib.breakout/breakout a-query stage-number expr)))(mu/defn breakouts :- [:maybe [:sequential ::lib.schema.expression/expression]]"Returns the list of breakouts on the target stage of `a-query`, or nil if there are none.**Code Health:** Healthy. This is a core API."- ([a-query] (breakouts a-query -1))+ ([a-query :- ::lib.schema/query] (breakouts a-query -1))([a-query :- ::lib.schema/querystage-number :- :int](lib.breakout/breakouts a-query stage-number)))@@ -611,7 +622,9 @@except that the breakout column has the `:temporal-unit` or `:binning` specified from the breakout clause.**Code Health:** Healthy."- ([a-query breakout-clause] (breakout-column a-query -1 breakout-clause))+ ([a-query :- ::lib.schema/query+ breakout-clause :- ::lib.schema.ref/ref]+ (breakout-column a-query -1 breakout-clause))([a-query :- ::lib.schema/querystage-number :- :intbreakout-clause :- ::lib.schema.ref/ref]@@ -639,7 +652,7 @@The removal is done properly per [[remove-clause]], so any references to the deleted breakouts will also be deleted.**Code Health:** Healthy."- ([a-query] (remove-all-breakouts a-query -1))+ ([a-query :- ::lib.schema/query] (remove-all-breakouts a-query -1))([a-query :- ::lib.schema/querystage-number :- :int](lib.breakout/remove-all-breakouts a-query stage-number)))@@ -666,17 +679,19 @@If `boolean-expression` is an exact duplicate of an existing filter, this silently does nothing.**Code Health:** Healthy. This is a core API."- ([a-query boolean-expression] (filter a-query -1 boolean-expression))+ ([a-query :- ::lib.schema/query+ boolean-expression :- [:or ::lib.schema.expression/boolean ::lib.schema.metadata/segment ::lib.schema.common/external-op]]+ (filter a-query -1 boolean-expression))([a-query :- ::lib.schema/querystage-number :- :int- boolean-expression :- some?]+ boolean-expression :- [:or ::lib.schema.expression/boolean ::lib.schema.metadata/segment ::lib.schema.common/external-op]](lib.filter/filter a-query stage-number boolean-expression)))(mu/defn filters :- [:maybe [:ref ::lib.schema/filters]]"Returns the list of filters on the target stage of `a-query`, or nil if there are none.**Code Health:** Healthy. This is a core API."- ([a-query] (filters a-query -1))+ ([a-query :- ::lib.schema/query] (filters a-query -1))([a-query :- ::lib.schema/querystage-number :- :int](lib.filter/filters a-query stage-number)))@@ -687,7 +702,7 @@logically equivalent to the conjunction of [[filters]].**Code Health:** Healthy. This is a core API."- ([a-query] (atomic-filters a-query -1))+ ([a-query :- ::lib.schema/query] (atomic-filters a-query -1))([a-query :- ::lib.schema/querystage-number :- :int](lib.filter/atomic-filters a-query stage-number)))@@ -705,11 +720,13 @@Supports an optional third argument `options`, which can have these options:- `:include-sensitive-fields? true` will return fields with `:visibility-type :sensitive`. (Default false.)"- ([a-query] (filterable-columns a-query -1))- ([a-query stage-number] (filterable-columns a-query stage-number nil))+ ([a-query :- ::lib.schema/query] (filterable-columns a-query -1))+ ([a-query :- ::lib.schema/query+ stage-number :- :int]+ (filterable-columns a-query stage-number nil))([a-query :- ::lib.schema/querystage-number :- :int- options :- [:maybe [:map [:include-sensitive-fields? :boolean]]]]+ options :- [:maybe [:map {:closed true} [:include-sensitive-fields? :boolean]]]](lib.filter/filterable-columns a-query stage-number options)));;; ### Building filters in code@@ -777,7 +794,9 @@**Code Health:** Healthy. This is a core API."- ([a-query a-join-or-joinable] (join a-query -1 a-join-or-joinable))+ ([a-query :- ::lib.schema/query+ a-join-or-joinable :- [:or ::lib.join.util/partial-join ::lib.join/joinable]]+ (join a-query -1 a-join-or-joinable))([a-query :- ::lib.schema/querystage-number :- :inta-join-or-joinable :- [:or ::lib.join.util/partial-join ::lib.join/joinable]]@@ -804,15 +823,21 @@- See also [[available-join-strategies]], since some engines only support certain kinds of joins.**Code Health:** Healthy. This is a core API."- ([a-joinable] (lib.join/join-clause a-joinable))- ([a-joinable conditions] (lib.join/join-clause a-joinable conditions))- ([a-joinable conditions strategy] (lib.join/join-clause a-joinable conditions strategy)))+ ([a-joinable :- ::lib.join/join-clause-source]+ (lib.join/join-clause a-joinable))+ ([a-joinable :- ::lib.join/join-clause-source+ conditions :- [:maybe ::lib.join/conditions-input]]+ (lib.join/join-clause a-joinable conditions))+ ([a-joinable :- ::lib.join/join-clause-source+ conditions :- [:maybe ::lib.join/conditions-input]+ strategy :- [:or ::lib.schema.join/strategy ::lib.schema.join/strategy.option]]+ (lib.join/join-clause a-joinable conditions strategy)))(mu/defn joins :- [:maybe ::lib.schema.join/joins]"Return the list of all explicit join clauses on the target stage of `a-query`, or nil if there are none.**Code Health:** Healthy. This is a core API."- ([a-query] (joins a-query -1))+ ([a-query :- ::lib.schema/query] (joins a-query -1))([a-query :- ::lib.schema/querystage-number :- :int](lib.join/joins a-query stage-number)))@@ -895,7 +920,7 @@**Code Health:** Healthy. This is a core API."[a-join :- ::lib.join.util/partial-join- conditions :- [:maybe [:sequential [:or ::lib.schema.expression/boolean ::lib.schema.common/external-op]]]]+ conditions :- [:maybe ::lib.join/conditions-input]](lib.join/with-join-conditions a-join conditions))(mu/defn suggested-join-conditions :- [:maybe [:sequential {:min 1} ::lib.schema.expression/boolean]]@@ -918,8 +943,13 @@If no plausible FK/PK pairs are found, this function returns nil and the join does not get a default condition.**Code Health:** Healthy. This is a core API."- ([a-query a-joinable] (suggested-join-conditions a-query -1 a-joinable))- ([a-query stage-number a-joinable] (suggested-join-conditions a-query stage-number a-joinable nil))+ ([a-query :- ::lib.schema/query+ a-joinable :- ::lib.join/join-or-joinable]+ (suggested-join-conditions a-query -1 a-joinable))+ ([a-query :- ::lib.schema/query+ stage-number :- :int+ a-joinable :- ::lib.join/join-or-joinable]+ (suggested-join-conditions a-query stage-number a-joinable nil))([a-query :- ::lib.schema/querystage-number :- :inta-joinable :- ::lib.join/join-or-joinable@@ -942,7 +972,10 @@**Code Health:** Healthy. This is the correct way to get the columns for software or a user to pick as the LHS ofa join condition."- ([a-query a-joinable lhs-expression-or-nil rhs-expression-or-nil]+ ([a-query :- ::lib.schema/query+ a-joinable :- [:maybe ::lib.join/join-or-joinable]+ lhs-expression-or-nil :- [:maybe ::lib.schema.expression/expression]+ rhs-expression-or-nil :- [:maybe ::lib.schema.expression/expression]](join-condition-lhs-columns a-query -1 a-joinable lhs-expression-or-nil rhs-expression-or-nil))([a-query :- ::lib.schema/querystage-number :- :int@@ -964,7 +997,10 @@**Code Health:** Healthy. This is the correct way to get the columns for software or a user to pick as the RHS ofa join condition."- ([a-query a-joinable lhs-expression-or-nil rhs-expression-or-nil]+ ([a-query :- ::lib.schema/query+ a-joinable :- ::lib.join/join-or-joinable+ lhs-expression-or-nil :- [:maybe ::lib.schema.expression/expression]+ rhs-expression-or-nil :- [:maybe ::lib.schema.expression/expression]](join-condition-rhs-columns a-query -1 a-joinable lhs-expression-or-nil rhs-expression-or-nil))([a-query :- ::lib.schema/querystage-number :- :int@@ -984,7 +1020,9 @@The returned operators are maps, so that they can support [[display-name]] etc.**Code Health:** Healthy."- ([a-query lhs-expression-or-nil rhs-expression-or-nil]+ ([a-query :- ::lib.schema/query+ lhs-expression-or-nil :- [:maybe ::lib.schema.expression/expression]+ rhs-expression-or-nil :- [:maybe ::lib.schema.expression/expression]](join-condition-operators a-query -1 lhs-expression-or-nil rhs-expression-or-nil))([a-query :- ::lib.schema/querystage-number :- :int@@ -1007,7 +1045,9 @@Passing an `option-or-unit` of nil will un-set the bucketing of both arguments.**Code Health:** Healthy."- ([a-query join-condition option-or-unit]+ ([a-query :- ::lib.schema/query+ join-condition :- [:or ::lib.schema.expression/boolean ::lib.schema.common/external-op]+ option-or-unit :- [:maybe [:or ::lib.schema.temporal-bucketing/option ::lib.schema.temporal-bucketing/unit]]](join-condition-update-temporal-bucketing a-query -1 join-condition option-or-unit))([a-query :- ::lib.schema/querystage-number :- :int@@ -1044,7 +1084,8 @@**Code Health:** Healthy. This is a core API."{:style/indent [:form]}[a-join :- ::lib.join.util/partial-join- fields :- [:maybe [:or [:enum :all :none] [:sequential some?]]]] ;; TODO: More precise schema.+ fields :- [:maybe [:or [:enum :all :none]+ [:sequential [:or ::lib.schema.metadata/column ::lib.schema.ref/ref]]]]](lib.join/with-join-fields a-join fields))(mu/defn with-join-source-fields :- ::lib.join.util/partial-join@@ -1058,7 +1099,7 @@**Code Health:** Healthy. This is a core API."[a-join :- ::lib.join.util/partial-join- cols :- [:maybe [:sequential some?]]] ; ideally [:sequential ::lib.schema.metadata/column]+ cols :- [:maybe [:sequential ::lib.schema.metadata/column]]](lib.join/with-join-source-fields a-join cols))(mu/defn join-fieldable-columns :- ::lib.metadata.calculation/visible-columns@@ -1072,7 +1113,9 @@are selected to be returned.**Code Health:** Healthy. This is a core API."- ([a-query a-joinable] (join-fieldable-columns a-query -1 a-joinable))+ ([a-query :- ::lib.schema/query+ a-joinable :- ::lib.join/join-or-joinable]+ (join-fieldable-columns a-query -1 a-joinable))([a-query :- ::lib.schema/querystage-number :- :inta-joinable :- ::lib.join/join-or-joinable]@@ -1089,9 +1132,12 @@If we can't determine a good display name for the LHS, this function returns \"Previous results\".**Code Health:** Healthy, Single Use. This exists to support UIs, but it's well-defined and safe to call."- ([a-query a-joinable]+ ([a-query :- ::lib.schema/query+ a-joinable :- [:maybe ::lib.join/join-or-joinable]](join-lhs-display-name a-query a-joinable nil))- ([a-query a-joinable condition-lhs-expression-or-nil]+ ([a-query :- ::lib.schema/query+ a-joinable :- [:maybe ::lib.join/join-or-joinable]+ condition-lhs-expression-or-nil :- [:maybe [:or ::lib.schema.metadata/column :mbql.clause/field]]](join-lhs-display-name a-query -1 a-joinable condition-lhs-expression-or-nil))([a-query :- ::lib.schema/querystage-number :- :int@@ -1128,10 +1174,7 @@Users of lib should be getting columns from [[filterable-columns]] et al, not manually setting join aliases on columnmetadata or refs."{:style/indent [:form]}- [field-or-join :- [:or- ::lib.schema.metadata/column- ::lib.join.util/partial-join- [:ref :mbql.clause/field]]+ [field-or-join :- ::lib.join/with-join-alias-targetjoin-alias :- [:maybe ::lib.schema.common/non-blank-string]](lib.join/with-join-alias field-or-join join-alias))@@ -1178,7 +1221,7 @@[[with-fields]]. [[fieldable-columns]] gives the list of columns which are valid to list here.**Code Health:** Healthy. This is a core API."- ([a-query] (fields a-query -1))+ ([a-query :- ::lib.schema/query] (fields a-query -1))([a-query :- ::lib.schema/querystage-number :- :int](lib.field/fields a-query stage-number)))@@ -1194,7 +1237,7 @@list is specified, then *all columns* are marked `:selected?`.**Code Health:** Healthy. This is a core API."- ([a-query] (fieldable-columns a-query -1))+ ([a-query :- ::lib.schema/query] (fieldable-columns a-query -1))([a-query :- ::lib.schema/querystage-number :- :int](lib.field/fieldable-columns a-query stage-number)))@@ -1234,7 +1277,9 @@stage return *everything* again.**Code Health:** Healthy. This is a core API."- ([a-query xs] (with-fields a-query -1 xs))+ ([a-query :- ::lib.schema/query+ xs :- [:maybe [:sequential [:or ::lib.schema.metadata/column ::lib.schema.ref/ref]]]]+ (with-fields a-query -1 xs))([a-query :- ::lib.schema/querystage-number :- :intxs :- [:maybe [:sequential [:or ::lib.schema.metadata/column ::lib.schema.ref/ref]]]]@@ -1286,13 +1331,19 @@- `:include-sensitive-fields?` returns fields the user has marked as *sensitive* in the Admin / Table Metadata screen.**Code Health:** Healthy. This is a core API."- ([a-query] (lib.metadata.calculation/returned-columns a-query))- ([a-query x] (lib.metadata.calculation/returned-columns a-query x))- ([a-query stage-number x] (lib.metadata.calculation/returned-columns a-query stage-number x))+ ([a-query :- ::lib.schema/query]+ (lib.metadata.calculation/returned-columns a-query))+ ([a-query :- ::lib.schema/query+ x :- ::lib.metadata.calculation/returned-columns-arg]+ (lib.metadata.calculation/returned-columns a-query x))([a-query :- ::lib.schema/querystage-number :- :int- x- options :- [:maybe ::lib.metadata.calculation/returned-columns.options]]+ x :- ::lib.metadata.calculation/returned-columns-arg]+ (lib.metadata.calculation/returned-columns a-query stage-number x))+ ([a-query :- ::lib.schema/query+ stage-number :- :int+ x :- ::lib.metadata.calculation/returned-columns-arg+ options :- [:maybe ::lib.metadata.calculation/visible-columns.options]](lib.metadata.calculation/returned-columns a-query stage-number x options)))(mu/defn visible-columns :- ::lib.metadata.calculation/visible-columns@@ -1307,8 +1358,11 @@**Code Health:** Deprecated. Should be unexported, and maybe removed outright. All callers should be calling[[filterable-columns]], [[breakoutable-columns]], etc. instead of this. If we really want to keep it, then its optionsshould be removed and it should be renamed to better reflect what it returns; perhaps `pre-aggregation-columns`."- ([a-query] (lib.metadata.calculation/visible-columns a-query))- ([a-query stage-number] (lib.metadata.calculation/visible-columns a-query stage-number))+ ([a-query :- ::lib.schema/query]+ (lib.metadata.calculation/visible-columns a-query))+ ([a-query :- ::lib.schema/query+ stage-number :- [:maybe :int]]+ (lib.metadata.calculation/visible-columns a-query stage-number))([a-query :- ::lib.schema/querystage-number :- :intoptions :- ::lib.metadata.calculation/visible-columns.options]@@ -1324,13 +1378,17 @@If you pass only `a-query`, the query is also used as `x`.**Code Health:** Healthy. This is a core API."- ([a-query] (display-name a-query a-query))- ([a-query x] (display-name a-query -1 x))- ([a-query stage-number x]+ ([a-query :- ::lib.schema/query] (display-name a-query a-query))+ ([a-query :- ::lib.schema/query+ x :- ::lib.metadata.calculation/displayable]+ (display-name a-query -1 x))+ ([a-query :- ::lib.schema/query+ stage-number :- :int+ x :- ::lib.metadata.calculation/displayable](lib.metadata.calculation/display-name a-query stage-number x))([a-query :- ::lib.schema/querystage-number :- :int- x+ x :- ::lib.metadata.calculation/displayablestyle :- ::lib.metadata.calculation/display-name-style](lib.metadata.calculation/display-name a-query stage-number x style)))@@ -1389,7 +1447,9 @@See [[describe-filter-operator]] to get a human-readable, translated description of the filter operator keywords.**Code Health:** Healthy. This is deliberately public, including the output format."- ([a-query a-filter-clause] (filter-parts a-query -1 a-filter-clause))+ ([a-query :- ::lib.schema/query+ a-filter-clause :- ::lib.schema.expression/boolean]+ (filter-parts a-query -1 a-filter-clause))([a-query :- ::lib.schema/querystage-number :- :inta-filter-clause :- ::lib.schema.expression/boolean]@@ -1404,7 +1464,7 @@- `:default` (the default, naturally) uses \"Is (not)\", \"Greater than\" and \"Less than\"- `:number` uses \"Equal to\" and \"Not equal to\"- `:temporal` uses \"After\" and \"Before\" for `:>` and `:<`"- ([operator] (describe-filter-operator operator :default))+ ([operator :- :keyword] (describe-filter-operator operator :default))([operator :- :keyword ; TODO: (bshepherdson 2026-03-06) This could be more specific, if the list were in a schema.variant :- [:enum :default :number :temporal]](lib.filter/describe-filter-operator operator variant)))diff --git a/src/metabase/lib/drill_thru.cljc b/src/metabase/lib/drill_thru.cljcindex 1fde89e3ee9..1268596b52d 100644--- a/src/metabase/lib/drill_thru.cljc+++ b/src/metabase/lib/drill_thru.cljc@@ -1,6 +1,7 @@(ns metabase.lib.drill-thru(:refer-clojure :exclude [select-keys not-empty #?(:clj for)])(:require+ [metabase.lib.common :as lib.common][metabase.lib.drill-thru.automatic-insights :as lib.drill-thru.automatic-insights][metabase.lib.drill-thru.column-extract :as lib.drill-thru.column-extract][metabase.lib.drill-thru.column-filter :as lib.drill-thru.column-filter]@@ -108,7 +109,8 @@For a value of `NULL` from the database, use the sentinel `:null`. Most of this file only cares whether the valuewas provided or not, but some things (eg. quick filters) treat `NULL` values differently.See [[metabase.lib.js/available-drill-thrus]]."- ([query context]+ ([query :- ::lib.schema/query+ context :- ::lib.schema.drill-thru/context](available-drill-thrus query -1 context))([query :- ::lib.schema/query@@ -153,14 +155,15 @@one of those returned by a call to [[available-drill-thrus]] with the same `query` and `stage-number`.Returns the updated query."- ([query drill]+ ([query :- ::lib.schema/query+ drill :- ::lib.schema.drill-thru/drill-thru](drill-thru query -1 nil drill))([query :- ::lib.schema/querystage-number :- :intcard-id :- [:maybe ::lib.schema.id/card]drill :- ::lib.schema.drill-thru/drill-thru- & args]+ & args :- [:* ::lib.common/op-arg]](log/debugf "Applying drill thru: %s" (:type drill))(let [{:keys [query stage-number]} (lib.query/wrap-native-query-with-mbql query stage-number card-id)](apply lib.drill-thru.common/drill-thru-method query stage-number drill args))))diff --git a/src/metabase/lib/drill_thru/common.cljc b/src/metabase/lib/drill_thru/common.cljcindex c3d06b8d122..aabd651b2cb 100644--- a/src/metabase/lib/drill_thru/common.cljc+++ b/src/metabase/lib/drill_thru/common.cljc@@ -164,11 +164,15 @@(mu/defn breakout->resolved-column :- ::lib.schema.metadata/column"Given a breakout sourced column, return the resolved metadata for the column in this stage."- ([query stage-number column] (breakout->resolved-column query stage-number column nil))+ ([query :- ::lib.schema/query+ stage-number :- :int+ column :- ::lib.schema.metadata/column]+ (breakout->resolved-column query stage-number column nil))([query :- ::lib.schema/querystage-number :- :intcolumn :- ::lib.schema.metadata/column- {:keys [preserve-type?]}]+ {:keys [preserve-type?]} :- [:maybe [:map {:closed true}+ [:preserve-type? {:optional true} :boolean]]]];; TODO: This is a hack to workaround field refs confusion that should be fixed by the field refs overhaul. Remove;; this function and possible-model-mapped-breakout-column?, above, once the field refs overhaul lands.;;diff --git a/src/metabase/lib/drill_thru/pivot.cljc b/src/metabase/lib/drill_thru/pivot.cljcindex 6e141b65fef..4d150397f0c 100644--- a/src/metabase/lib/drill_thru/pivot.cljc+++ b/src/metabase/lib/drill_thru/pivot.cljc@@ -161,14 +161,12 @@(mu/defn pivot-types :- [:sequential ::lib.schema.drill-thru/pivot-types]"A helper for the FE. Returns the set of pivot types (category, location, time) that apply to this drill-thru."- [drill-thru :- [:and ::lib.schema.drill-thru/drill-thru- [:map [:type [:= :drill-thru/pivot]]]]]+ [drill-thru :- ::lib.schema.drill-thru/drill-thru.pivot](-> drill-thru :pivots keys sort))(mu/defn pivot-columns-for-type :- [:sequential ::lib.schema.metadata/column]"A helper for the FE. Returns all the columns of the given type which can be used to pivot the query."- [drill-thru :- [:and ::lib.schema.drill-thru/drill-thru- [:map [:type [:= :drill-thru/pivot]]]]+ [drill-thru :- ::lib.schema.drill-thru/drill-thru.pivotpivot-type :- ::lib.schema.drill-thru/pivot-types](get-in drill-thru [:pivots pivot-type]))diff --git a/src/metabase/lib/drill_thru/quick_filter.cljc b/src/metabase/lib/drill_thru/quick_filter.cljcindex b125c198c7e..334c055c95a 100644--- a/src/metabase/lib/drill_thru/quick_filter.cljc+++ b/src/metabase/lib/drill_thru/quick_filter.cljc@@ -52,6 +52,7 @@[metabase.lib.schema.common :as lib.schema.common][metabase.lib.schema.drill-thru :as lib.schema.drill-thru][metabase.lib.schema.expression :as lib.schema.expression]+ [metabase.lib.schema.literal :as lib.schema.literal][metabase.lib.schema.metadata :as lib.schema.metadata][metabase.lib.temporal-bucket :as lib.temporal-bucket][metabase.lib.types.isa :as lib.types.isa]@@ -71,7 +72,7 @@(mu/defn- operators-for :- [:sequential ::lib.schema.drill-thru/drill-thru.quick-filter.operator][column :- ::lib.schema.metadata/column- value]+ value :- [:maybe [:or [:= :null] ::lib.schema.literal/param-value]]](let [field-ref (lib.ref/ref column)](cond(lib.types.isa/structured? column)diff --git a/src/metabase/lib/drill_thru/sort.cljc b/src/metabase/lib/drill_thru/sort.cljcindex b57928db63a..1e91cd2690c 100644--- a/src/metabase/lib/drill_thru/sort.cljc+++ b/src/metabase/lib/drill_thru/sort.cljc@@ -25,6 +25,7 @@[metabase.lib.order-by :as lib.order-by][metabase.lib.schema :as lib.schema][metabase.lib.schema.drill-thru :as lib.schema.drill-thru]+ [metabase.lib.schema.metadata :as lib.schema.metadata][metabase.lib.schema.order-by :as lib.schema.order-by][metabase.lib.types.isa :as lib.types.isa][metabase.util.malli :as mu]))@@ -38,13 +39,17 @@(lib.order-by/orderable-columns query stage-number)))(mu/defn- existing-order-by-clause :- [:maybe ::lib.schema.order-by/order-by]- [query stage-number column]+ [query :- ::lib.schema/query+ stage-number :- :int+ column :- ::lib.schema.metadata/column](m/find-first (fn [[_direction _opts expr, :as _asc-or-desc-clause]](lib.equality/find-matching-column query stage-number expr [column]))(lib.order-by/order-bys query stage-number)))(mu/defn- existing-order-by-direction :- [:maybe ::lib.schema.order-by/direction]- [query stage-number column]+ [query :- ::lib.schema/query+ stage-number :- :int+ column :- ::lib.schema.metadata/column](when-let [[direction _opts _expr] (existing-order-by-clause query stage-number column)]direction))diff --git a/src/metabase/lib/drill_thru/underlying_records.cljc b/src/metabase/lib/drill_thru/underlying_records.cljcindex 7e9c30db16f..ba072a33b2e 100644--- a/src/metabase/lib/drill_thru/underlying_records.cljc+++ b/src/metabase/lib/drill_thru/underlying_records.cljc@@ -41,6 +41,7 @@[metabase.lib.options :as lib.options][metabase.lib.schema :as lib.schema][metabase.lib.schema.drill-thru :as lib.schema.drill-thru]+ [metabase.lib.schema.literal :as lib.schema.literal][metabase.lib.schema.metadata :as lib.schema.metadata][metabase.lib.schema.ref :as lib.schema.ref][metabase.lib.schema.temporal-bucketing :as lib.schema.temporal-bucketing]@@ -140,7 +141,10 @@stage-number :- :intcolumn :- ::lib.schema.metadata/columncolumn-ref :- ::lib.schema.ref/ref- value :- :any]+ value :- [:maybe [:or+ ::lib.schema.literal/param-value+ [:sequential ::lib.schema.literal/param-value]+ #?@(:cljs [[:fn {:error/message "JS array"} array?]])]]](let [filter-column (lib.drill-thru.common/breakout->filterable-column query stage-number column-ref column)values (non-empty-seq value)filter-clauses (ordiff --git a/src/metabase/lib/equality.cljc b/src/metabase/lib/equality.cljcindex 9bef6dc5eb1..063e7ddeff1 100644--- a/src/metabase/lib/equality.cljc+++ b/src/metabase/lib/equality.cljc@@ -317,7 +317,7 @@ are known to be the same."{:ref a-ref:columns columns}))-(mu/defn- expression-column? [column]+(mu/defn- expression-column? [column :- ::lib.schema.metadata/column](or (clojure.core/= (:lib/source column) :source/expressions)(:lib/expression-name column)))@@ -416,7 +416,7 @@ are known to be the same."(disambiguate-matches-ignoring-join-alias a-ref columns))))(def ^:private FindMatchingColumnOptions- [:map [:generous? {:optional true} :boolean]])+ [:map {:closed true} [:generous? {:optional true} :boolean]])(mu/defn find-matching-column :- [:maybe ::lib.schema.metadata/column]"Given `a-ref-or-column` and a list of `columns`, finds the column that best matches this ref or column.@@ -442,7 +442,8 @@ are known to be the same."- The same disambiguation (by `:join-alias` etc.) is applied if there are multiple plausible matches.Returns the matching column, or nil if no match is found."- ([a-ref columns]+ ([a-ref :- ::lib.schema.ref/ref+ columns :- [:sequential ::lib.schema.metadata/column]](find-matching-column a-ref columns {}))([[ref-kind _opts ref-id :as a-ref] :- ::lib.schema.ref/ref@@ -464,7 +465,10 @@ are known to be the same."(disambiguate-matches a-ref plausible)))(throw (ex-info "Unknown type of ref" {:ref a-ref}))))- ([query stage-number a-ref-or-column columns]+ ([query :- [:maybe ::lib.schema/query]+ stage-number :- :int+ a-ref-or-column :- [:or ::lib.schema.metadata/column ::lib.schema.ref/ref]+ columns :- [:sequential ::lib.schema.metadata/column]](find-matching-column query stage-number a-ref-or-column columns {}))([query :- [:maybe ::lib.schema/query]@@ -574,12 +578,14 @@ are known to be the same."(mu/defn find-column-for-legacy-ref :- [:maybe ::lib.schema.metadata/column]"Like [[find-matching-column]], but takes a legacy MBQL reference. The name here is for consistency with otherFE names for similar functions."- ([query legacy-ref metadatas]+ ([query :- ::lib.schema/query+ legacy-ref :- :metabase.lib.convert/unnormalized-legacy-ref+ metadatas :- [:maybe [:sequential ::lib.schema.metadata/column]]](find-column-for-legacy-ref query -1 legacy-ref metadatas))([query :- ::lib.schema/querystage-number :- :int- legacy-ref :- :some+ legacy-ref :- :metabase.lib.convert/unnormalized-legacy-refmetadatas :- [:maybe [:sequential ::lib.schema.metadata/column]]](find-matching-column query stage-number (lib.convert/legacy-ref->mbql5 query stage-number legacy-ref) metadatas)))diff --git a/src/metabase/lib/expression.cljc b/src/metabase/lib/expression.cljcindex cb410a0f9ca..94872d757a6 100644--- a/src/metabase/lib/expression.cljc+++ b/src/metabase/lib/expression.cljc@@ -47,7 +47,8 @@(mu/defn maybe-resolve-expression :- ::lib.schema.expression/expression"Find the expression with `expression-name` in a given stage of a `query`, or nil if it doesn't exist."- ([query expression-name]+ ([query :- ::lib.schema/query+ expression-name :- ::lib.schema.common/non-blank-string](maybe-resolve-expression query -1 expression-name))([query :- ::lib.schema/query@@ -60,7 +61,8 @@(mu/defn resolve-expression :- ::lib.schema.expression/expression"Find the expression with `expression-name` in a given stage of a `query`, or throw an Exception if it doesn'texist."- ([query expression-name]+ ([query :- ::lib.schema/query+ expression-name :- ::lib.schema.common/non-blank-string](resolve-expression query -1 expression-name))([query :- ::lib.schema/query@@ -298,10 +300,17 @@(some #(-> % :name u/lower-case-en (= expr-name)) cols)))(mr/def ::add-expression-options- [:map+ [:map {:closed true};; default: true[:add-to-fields? {:optional true} [:maybe :boolean]]])+(mr/def ::expressionable+ "Schema for something you can pass to [[expression]] to add to a query as an expression."+ [:or+ [:ref ::lib.schema.expression/expression]+ [:ref ::lib.schema.common/external-op]+ [:ref ::lib.schema.metadata/column]])+(defn- add-expression-to-stage[stageexpression@@ -320,16 +329,21 @@* `:add-to-fields?` (default: `true`) -- whether to add an `:expression` ref to `:fields` if one is present in thequery."- ([query expression-name expressionable]+ ([query :- ::lib.schema/query+ expression-name :- ::lib.schema.common/non-blank-string+ expressionable :- ::expressionable](expression query -1 expression-name expressionable))- ([query stage-number expression-name expressionable]+ ([query :- ::lib.schema/query+ stage-number :- [:maybe :int]+ expression-name :- ::lib.schema.common/non-blank-string+ expressionable :- ::expressionable](expression query stage-number expression-name expressionable nil))([query :- ::lib.schema/querystage-number :- [:maybe :int]expression-name :- ::lib.schema.common/non-blank-string- expressionable+ expressionable :- ::expressionableoptions :- [:maybe ::add-expression-options]](let [stage-number (or stage-number -1)expressionable (lib.common/->op-arg expressionable)]@@ -405,9 +419,10 @@(mu/defn datetime :- :mbql.clause/datetime"Create a standalone clause of type `datetime`."- ([value]+ ([value :- ::lib.common/op-arg](lib.common/defop-create :datetime [value]))- ([value mode]+ ([value :- ::lib.common/op-arg+ mode :- (into [:enum] lib.schema.expression.temporal/datetime-modes)](into [:datetime {:lib/uuid (str (random-uuid)):mode mode}](map lib.common/->op-arg) [value])))@@ -440,7 +455,7 @@[:map[:lib/uuid {:optional true} ::lib.schema.common/uuid][:effective-type {:optional true} ::lib.schema.common/base-type]]]]- literal]+ literal :- [:ref ::lib.schema.literal/value.value]](let [base-type (or (:base-type opts) (lib.schema.expression/type-of-resolved literal))effective-type (or (:effective-type opts) base-type)](-> [:value@@ -470,7 +485,7 @@(mu/defn expressions-metadata :- [:maybe [:sequential ::lib.schema.metadata/column]]"Get metadata about the expressions in a given stage of a `query`."- ([query]+ ([query :- ::lib.schema/query](expressions-metadata query -1))([query :- ::lib.schema/query@@ -480,7 +495,7 @@(mu/defn expressions :- [:maybe ::lib.schema.expression/expressions]"Get the expressions map from a given stage of a `query`."- ([query]+ ([query :- ::lib.schema/query](expressions query -1))([query :- ::lib.schema/query@@ -541,7 +556,8 @@(-> (lib/query ...)(lib/expression \"My Expression\" ...)(as-> <> (lib/aggregate <> (lib/avg (lib/expression-ref <> \"My Expression\")))))"- ([query expression-name]+ ([query :- ::lib.schema/query+ expression-name :- ::lib.schema.common/non-blank-string](expression-ref query -1 expression-name))([query :- ::lib.schema/query@@ -706,6 +722,10 @@(i18n/tru "Types are incompatible: {0} expects {1} as the {2} parameter." op-name type-desc param-pos)(i18n/tru "Types are incompatible."))))+(mr/def ::diagnosable-expression+ "An expression, aggregation, or filter that may be invalid; [[diagnose-expression]] reports what is wrong with it."+ [:schema {::mr/deliberately-open true, :description "a possibly invalid MBQL expression"} :any])+(mu/defn diagnose-expression :- [:maybe [:map [:message :string]]]"Checks `expr` for type errors and, if `expression-mode` is :expression and`expression-position` is provided, for cyclic references with other expressions.@@ -725,7 +745,7 @@[query :- ::lib.schema/querystage-number :- :intexpression-mode :- [:enum :expression :aggregation :filter]- expr :- :any+ expr :- ::diagnosable-expressionexpression-position :- [:maybe :int]](binding [lib.schema.expression/*suppress-expression-type-check?* false](let [explainer (clojure.core/case expression-modediff --git a/src/metabase/lib/fe_util.cljc b/src/metabase/lib/fe_util.cljcindex 0ddae3e91d9..64b1d671d39 100644--- a/src/metabase/lib/fe_util.cljc+++ b/src/metabase/lib/fe_util.cljc@@ -17,6 +17,7 @@[metabase.lib.query :as lib.query][metabase.lib.ref :as lib.ref][metabase.lib.schema :as lib.schema]+ [metabase.lib.schema.common :as lib.schema.common][metabase.lib.schema.expression :as lib.schema.expression][metabase.lib.schema.filter :as lib.schema.filter][metabase.lib.schema.id :as lib.schema.id]@@ -46,13 +47,21 @@::lib.schema.metadata/segment::lib.schema.metadata/metric])+(def ^:private ExpressionOptions+ "Like `:metabase.lib.schema.common/options`, but `:lib/uuid` need not be set yet;+ see [[metabase.lib.options/ensure-uuid]]."+ [:merge+ ::lib.schema.common/options+ [:map {:closed true}+ [:lib/uuid {:optional true} ::lib.schema.common/uuid]]])+(def ^:private ExpressionParts[:schema{:registry {::expression-parts- [:map+ [:map {:closed true}[:lib/type [:= :mbql/expression-parts]][:operator [:or :keyword :string]]- [:options :map]+ [:options ExpressionOptions][:args [:sequential [:or ExpressionArg [:ref ::expression-parts]]]]]}}::expression-parts])@@ -218,7 +227,8 @@(mu/defn expression-parts :- [:or ExpressionArg ExpressionParts]"Return the parts of the filter clause `arg` in query `query` at stage `stage-number`."- ([query value]+ ([query :- ::lib.schema/query+ value :- [:or ::lib.schema.expression/expression ExpressionArg ExpressionParts]](expression-parts query -1 value))([query :- ::lib.schema/query@@ -288,7 +298,7 @@([operator :- [:or :keyword :string]args :- [:sequential [:or ExpressionArg ExpressionParts ::lib.schema.expression/expression]]- options :- [:maybe :map]]+ options :- [:maybe ExpressionOptions]](expression-clause-method {:lib/type :mbql/expression-parts:operator operator:options options@@ -656,7 +666,9 @@(mu/defn exclude-date-filter-parts :- [:maybe ExcludeDateFilterParts]"Destructures an exclude date filter clause created by [[exclude-date-filter-clause]]. Returns `nil` if the clausedoes not match the expected shape."- [query stage-number filter-clause]+ [query :- ::lib.schema/query+ stage-number :- :int+ filter-clause :- ::lib.schema.expression/expression](let [ref->col #(column-metadata-from-ref query stage-number %)date-col? #(ref-clause-with-type? % [:type/Date :type/DateTime])op->unit {:get-hour :hour-of-day@@ -788,7 +800,9 @@Can be expanded as needed but only currently defined for a narrow set of date filters.Falls back to the full filter display-name"- [query stage-number filter-clause]+ [query :- ::lib.schema/query+ stage-number :- :int+ filter-clause :- ::lib.schema.expression/expression](let [->temporal-name #(u.time/format-unit % nil)temporal? #(lib.util/original-isa? % :type/Temporal)unit= (fn [maybe-clause unit-or-units]@@ -841,7 +855,7 @@(i18n/tru "Is Not Empty")[:time-interval opts (_ :guard temporal?) n unit]- (lib.temporal-bucket/describe-temporal-interval n unit opts)+ (lib.temporal-bucket/describe-temporal-interval n unit (select-keys opts [:include-current]))[:relative-time-interval _ (_ :guard temporal?) n unit offset offset-unit](lib.temporal-bucket/describe-temporal-interval-with-offset n unit offset offset-unit)diff --git a/src/metabase/lib/field.cljc b/src/metabase/lib/field.cljcindex 5b8f26fb1b6..496dea84dc2 100644--- a/src/metabase/lib/field.cljc+++ b/src/metabase/lib/field.cljc@@ -523,16 +523,17 @@Throws if narrowing the projection would strand a reference in a later stage (a column that stage no longerreturns)."- ([xs]+ ([xs :- [:maybe [:sequential ::lib.ref/referenceable]]](fn [query stage-number](with-fields query stage-number xs)))- ([query xs]+ ([query :- ::lib.schema/query+ xs :- [:maybe [:sequential ::lib.ref/referenceable]]](with-fields query -1 xs))([query :- ::lib.schema/querystage-number :- :int- xs]+ xs :- [:maybe [:sequential ::lib.ref/referenceable]]](let [xs (not-empty (mapv lib.ref/ref xs));; If any fields are specified, include all expressions not yet included.expr-cols (expression-columns query stage-number)@@ -553,7 +554,7 @@(mu/defn fields :- [:maybe [:ref ::lib.schema/fields]]"Fetches the `:fields` for a query. Returns `nil` if there are no `:fields`. `:fields` should never be empty; this isenforced by the Malli schema."- ([query]+ ([query :- ::lib.schema/query](fields query -1))([query :- ::lib.schema/query@@ -567,7 +568,7 @@Includes a `:selected?` key letting you know this column is already in `:fields` or not; if `:fields` isunspecified, all these columns are returned by default, so `:selected?` is true for all columns (this is a littlestrange but it matches the behavior of the QB UI)."- ([query]+ ([query :- ::lib.schema/query](fieldable-columns query -1))([query :- ::lib.schema/query@@ -762,12 +763,13 @@(mu/defn find-visible-column-for-ref :- [:maybe ::lib.schema.metadata/column]"Return the visible column in `query` at `stage-number` referenced by `field-ref`. If `stage-number` is omitted, thelast stage is used. This is currently only meant for use with `:field` clauses."- ([query field-ref]+ ([query :- ::lib.schema/query+ field-ref :- [:or ::lib.schema.metadata/column ::lib.schema.ref/ref]](find-visible-column-for-ref query -1 field-ref))([query :- ::lib.schema/querystage-number :- :int- field-ref :- some?]+ field-ref :- [:or ::lib.schema.metadata/column ::lib.schema.ref/ref]](let [;; not 100% sure why, but [[lib.metadata.calculation/visible-columns]] doesn't seem to return aggregations,;; so we have to use [[lib.metadata.calculation/returned-columns]] instead.columns ((if (= (lib.dispatch/dispatch-value field-ref) :aggregation)@@ -801,13 +803,12 @@Note that this value is not necessarily the same as the value of `has_field_values` in the application database.`has_field_values` may be unset, in which case we will try to infer it. `:auto-list` is not currently understood bythe FE filter stuff, so we will instead return `:list`; the distinction is not important to it anyway."- [{:keys [has-field-values], :as field} :- [:map- ;; this doesn't use `::lib.schema.metadata/column` because it's stricter- ;; than we need and the REST API calls this function with optimized Field- ;; maps that don't include some keys like `:name`- [:base-type {:optional true} [:maybe ::lib.schema.common/base-type]]- [:effective-type {:optional true} [:maybe ::lib.schema.common/base-type]]- [:has-field-values {:optional true} [:maybe ::lib.schema.metadata/column.has-field-values]]]]+ [{:keys [has-field-values], :as field} :- [:or+ ::lib.schema.metadata/column+ [:map {:closed true}+ [:base-type {:optional true} [:maybe ::lib.schema.common/base-type]]+ [:effective-type {:optional true} [:maybe ::lib.schema.common/base-type]]+ [:has-field-values {:optional true} [:maybe ::lib.schema.metadata/column.has-field-values]]]]](cond;; if `has_field_values` is set in the DB, use that value; but if it's `auto-list`, return the value as `list` to;; avoid confusing FE code, which can remain blissfully unaware that `auto-list` is a thingdiff --git a/src/metabase/lib/field/resolution.cljc b/src/metabase/lib/field/resolution.cljcindex d8edc061609..857a670810f 100644--- a/src/metabase/lib/field/resolution.cljc+++ b/src/metabase/lib/field/resolution.cljc@@ -350,14 +350,6 @@(log/debug "Unable to resolve in previous stage =(")nil))))))-(mr/def ::source-field-info- "The subset of field ref options that identify which implicit join a ref refers to: `:source-field` (the FK field ID,- required), and optionally `:source-field-name` and `:source-field-join-alias` for disambiguation."- [:map- [:source-field ::lib.schema.id/field]- [:source-field-name {:optional true} :string]- [:source-field-join-alias {:optional true} :string]])-(mu/defn- resolve-in-implicit-join-previous-stage :- [:maybe ::lib.metadata.calculation/visible-column]"First, try to resolve the implicit join from the previous stage columns -- the join might have already beenperformed there and `:source-field` was specified incorrectly. (You're only supposed to specify this in the stage@@ -368,7 +360,7 @@renaming."[query :- ::lib.schema/querystage-number :- :int- source-field-info :- ::source-field-info+ source-field-info :- ::lib.schema.ref/field.optionsid-or-name :- ::id-or-name](when-some [previous-stage-number (lib.util/previous-stage-number query stage-number)](let [{:keys [source-field source-field-name source-field-join-alias]} source-field-info@@ -392,7 +384,9 @@"Find the reified implicit join (i.e., a join added bythe [[metabase.query-processor.middleware.add-implicit-joins]] middleware) that has `:fk-field-id` if one exists;returns tuple of `[join join-stage-number]`."- [query stage-number source-field-id]+ [query :- ::lib.schema/query+ stage-number :- :int+ source-field-id :- ::lib.schema.id/field](or (when-some [join (m/find-first (fn [join](= (:fk-field-id join) source-field-id))(:joins (lib.util/query-stage query stage-number)))]@@ -473,7 +467,7 @@(mu/defn- resolve-in-implicit-join :- [:maybe ::lib.metadata.calculation/visible-column][query :- ::lib.schema/querystage-number :- :int- source-field-info :- ::source-field-info+ source-field-info :- ::lib.schema.ref/field.optionsid-or-name :- ::id-or-name](let [source-field-id (:source-field source-field-info)](log/debugf "Resolving implicitly joined %s (source Field ID = %s) in stage %s"@@ -555,7 +549,9 @@:lib/source-column-alias id-or-name})))(mu/defn- resolve-from-previous-stage-or-source* :- [:maybe ::lib.metadata.calculation/visible-column]- [query stage-number id-or-name]+ [query :- ::lib.schema/query+ stage-number :- :int+ id-or-name :- ::id-or-name](b/cond:let [stage (lib.util/query-stage query stage-number)source-table-id (:source-table stage)]@@ -592,7 +588,9 @@(mu/defn- resolve-ref-missing-join-alias :- [:maybe ::lib.metadata.calculation/visible-column]"Try finding a match in joins (field ref is missing `:join-alias`)."- [query stage-number id-or-name]+ [query :- ::lib.schema/query+ stage-number :- :int+ id-or-name :- ::id-or-name](log/debugf "Assuming %s is from a join, and missing :join-alias" (pr-str id-or-name))(or (when (string? id-or-name)(let [parts (str/split id-or-name #"__" 2)]diff --git a/src/metabase/lib/field/util.cljc b/src/metabase/lib/field/util.cljcindex 193d772c36e..d2833cd8273 100644--- a/src/metabase/lib/field/util.cljc+++ b/src/metabase/lib/field/util.cljc@@ -30,8 +30,7 @@(mu/defn inherited-column? :- :boolean"Is the `column` coming directly from a card, a native query, or a previous query stage?"- [column :- [:map- [:lib/source {:optional true} [:maybe ::lib.schema.metadata/column.source]]]]+ [column :- ::lib.schema.metadata/column](some? (#{:source/card :source/native :source/previous-stage} (:lib/source column))))(mu/defn inherited-column-name :- [:maybe :string]@@ -125,8 +124,7 @@stage' to the equivalent keys that mean 'this happened at some stage in the past' e.g.`:lib/join-alias` and `:lib/expression-name` become `:lib/original-join-alias` and`:lib/original-expression-name` respectively."- [col :- [:map- [:lib/type [:= :metadata/column]]]]+ [col :- ::lib.schema.metadata/column.map](-> col(set/rename-keys {:fk-field-id :lib/original-fk-field-id:fk-field-name :lib/original-fk-field-namediff --git a/src/metabase/lib/filter.cljc b/src/metabase/lib/filter.cljcindex df190181396..28217f3a64a 100644--- a/src/metabase/lib/filter.cljc+++ b/src/metabase/lib/filter.cljc@@ -346,10 +346,10 @@(clojure.core/= unit :day)))((binary-filter-display-fns :is)(lib.metadata.calculation/display-name query stage-number expr style)- (u/lower-case-en (lib.temporal-bucket/describe-temporal-interval n unit opts)))+ (u/lower-case-en (lib.temporal-bucket/describe-temporal-interval n unit (perf/select-keys opts [:include-current]))))((binary-filter-display-fns :is-in-the)(lib.metadata.calculation/display-name query stage-number expr style)- (u/lower-case-en (lib.temporal-bucket/describe-temporal-interval n unit opts)))))+ (u/lower-case-en (lib.temporal-bucket/describe-temporal-interval n unit (perf/select-keys opts [:include-current]))))))(defmethod lib.metadata.calculation/display-name-method :relative-time-interval[query stage-number [_tag _opts column value bucket offset-value offset-bucket] style]@@ -447,12 +447,12 @@(mu/defn filter :- :metabase.lib.schema/query"Sets `boolean-expression` as a filter on `query`. Ignores duplicate filters (ignoring :lib/uuid)."([query :- :metabase.lib.schema/query- boolean-expression]+ boolean-expression :- ::lib.common/op-arg](metabase.lib.filter/filter query nil boolean-expression))([query :- :metabase.lib.schema/querystage-number :- [:maybe :int]- boolean-expression]+ boolean-expression :- ::lib.common/op-arg];; if this is a Segment metadata, convert it to `:segment` MBQL clause before adding(if (clojure.core/= (lib.dispatch/dispatch-value boolean-expression) :metadata/segment)(recur query stage-number (lib.ref/ref boolean-expression))@@ -550,7 +550,7 @@a `column`, and arguments."[filter-operator :- [:or ::lib.schema.filter/operator :keyword :string]column :- ::lib.schema.metadata/column- & args]+ & args :- [:* ::lib.common/op-arg]](let [tag (if (map? filter-operator)(:short filter-operator)(keyword filter-operator))]@@ -568,7 +568,8 @@(mu/defn filter-parts :- ::filter-parts"Return the parts of the filter clause `a-filter-clause` in query `query` at stage `stage-number`.Might obsolete [[filter-operator]]."- ([query a-filter-clause]+ ([query :- ::lib.schema/query+ a-filter-clause :- ::lib.schema.expression/boolean](filter-parts query -1 a-filter-clause))([query :- ::lib.schema/querydiff --git a/src/metabase/lib/filter/desugar.cljc b/src/metabase/lib/filter/desugar.cljcindex f7fa5272303..09180332010 100644--- a/src/metabase/lib/filter/desugar.cljc+++ b/src/metabase/lib/filter/desugar.cljc@@ -8,6 +8,7 @@[metabase.lib.filter :as lib.filter][metabase.lib.filter.simplify-compound :as lib.filter.simplify-compound][metabase.lib.options :as lib.options]+ [metabase.lib.schema.common :as lib.schema.common][metabase.lib.schema.expression :as lib.schema.expression][metabase.lib.schema.mbql-clause :as lib.schema.mbql-clause][metabase.lib.schema.temporal-bucketing :as lib.schema.temporal-bucketing]@@ -299,7 +300,7 @@(mu/defn- temporal-case-expression :- :mbql.clause/case"Creates a `:case` expression with a condition for each value of the given unit."[expr :- ::clause- opts :- :map+ opts :- ::lib.schema.common/optionsunit :- :keywordn :- :int](let [user-locale #?(:clj (i18n/user-locale)diff --git a/src/metabase/lib/filter/simplify_compound.cljc b/src/metabase/lib/filter/simplify_compound.cljcindex f9298cb2153..5079c597c9b 100644--- a/src/metabase/lib/filter/simplify_compound.cljc+++ b/src/metabase/lib/filter/simplify_compound.cljc@@ -4,6 +4,8 @@[medley.core :as m][metabase.lib.filter :as lib.filter][metabase.lib.options :as lib.options]+ [metabase.lib.schema.common :as lib.schema.common]+ [metabase.lib.schema.expression :as lib.schema.expression][metabase.lib.schema.mbql-clause :as lib.schema.mbql-clause][metabase.lib.schema.util :as lib.schema.util][metabase.lib.util :as lib.util]@@ -17,8 +19,8 @@[:schema[:catkeyword?- map?- [:+ any?]]])+ ::lib.schema.common/options+ [:* [:maybe [:or ::lib.schema.expression/expression [:ref ::mbql-clause]]]]]])(mu/defn- combine-compound-filters-of-type :- [:sequential [:maybe ::mbql-clause]][tag :- [:enum :and :or]@@ -43,8 +45,8 @@(mu/defn- simplify-and-or-filter :- [:maybe ::lib.schema.mbql-clause/clause][tag :- [:enum :and :or]- opts :- :map- args]+ opts :- ::lib.schema.common/options+ args :- [:sequential [:maybe ::mbql-clause]]](let [args (m/distinct-by lib.schema.util/mbql-clause-distinct-key (filter some? args))](case (count args);; an empty filter, toss it@@ -63,11 +65,15 @@;; there is a change, we might be able to simplify even further(recur tag opts simplified)))))))+(mr/def ::simplifiable+ "A filter clause, possibly malformed (e.g. with `nil` args), or any form [[simplify-compound-filter]] walks to find them."+ [:schema {::mr/deliberately-open true, :description "a form containing possibly malformed compound filters"} :any])+(mu/defn simplify-compound-filter :- ::lib.schema.util/unique-uuids"Simplify compound `:and`, `:or`, and `:not` compound filters, combining or eliminating them where possible. Thisalso fixes theoretically disallowed compound filters like `:and` with only a single subclause, and eliminates `nils`and duplicate subclauses from the clauses."- [x]+ [x :- ::simplifiable](match/replace x;; double negation, eliminate both[:not opts [:not arg-opts arg-arg]]diff --git a/src/metabase/lib/filter/update.cljc b/src/metabase/lib/filter/update.cljcindex 68d766107ed..ae55d635e7c 100644--- a/src/metabase/lib/filter/update.cljc+++ b/src/metabase/lib/filter/update.cljc@@ -70,7 +70,10 @@(mu/defn update-numeric-filter :- ::lib.schema/query"Add or update a filter against `numeric-column`. Adapted fromhttps://github.com/metabase/metabase/blob/98bcd7fc3102bd7c07e8b68878c3738f3cb8727b/frontend/src/metabase-lib/queries/utils/actions.js#L151-L154"- ([query numeric-column start end]+ ([query :- ::lib.schema/query+ numeric-column :- ::lib.schema.metadata/column+ start :- number?+ end :- number?](update-numeric-filter query -1 numeric-column start end))([query :- ::lib.schema/query@@ -174,7 +177,10 @@This is adapted from old MLv1 code herehttps://github.com/metabase/metabase/blob/98bcd7fc3102bd7c07e8b68878c3738f3cb8727b/frontend/src/metabase-lib/queries/utils/actions.js#L75-L132"- ([query temporal-column start end]+ ([query :- ::lib.schema/query+ temporal-column :- ::lib.schema.metadata/column+ start :- ::temporal-literal+ end :- ::temporal-literal](update-temporal-filter query -1 temporal-column start end))([query :- ::lib.schema/query@@ -220,7 +226,7 @@(lib.filter/filter query stage-number (lib.filter/between temporal-column start end))))))))(mr/def ::lat-lon.bounds- [:map+ [:map {:closed true}[:north number?][:east number?][:south number?]@@ -229,13 +235,16 @@(mu/defn update-lat-lon-filter :- ::lib.schema/query"For use powering the brush zoom-in behavior in map visualizations. Adapted fromhttps://github.com/metabase/metabase/blob/98bcd7fc3102bd7c07e8b68878c3738f3cb8727b/frontend/src/metabase-lib/queries/utils/actions.js#L134-L149"- ([query latitude-column longitude-column bounds]+ ([query :- ::lib.schema/query+ latitude-column :- ::lib.schema.metadata/column+ longitude-column :- ::lib.schema.metadata/column+ bounds :- [:ref ::lat-lon.bounds]](update-lat-lon-filter query -1 latitude-column longitude-column bounds))([query :- ::lib.schema/querystage-number :- :intlatitude-column :- ::lib.schema.metadata/column- longitude-column :- :some+ longitude-column :- ::lib.schema.metadata/column{:keys [north east south west], :as _bounds} :- [:ref ::lat-lon.bounds]](-> query(remove-existing-filters-against-column-checking-subclauses stage-number latitude-column)diff --git a/src/metabase/lib/join.cljc b/src/metabase/lib/join.cljcindex eb9c84b901f..aa246b3ded7 100644--- a/src/metabase/lib/join.cljc+++ b/src/metabase/lib/join.cljc@@ -24,6 +24,7 @@[metabase.lib.schema.expression :as lib.schema.expression][metabase.lib.schema.join :as lib.schema.join][metabase.lib.schema.metadata :as lib.schema.metadata]+ [metabase.lib.schema.ref :as lib.schema.ref][metabase.lib.schema.temporal-bucketing :as lib.schema.temporal-bucketing][metabase.lib.stage.util :as lib.stage.util][metabase.lib.temporal-bucket :as lib.temporal-bucket]@@ -49,6 +50,9 @@[:ref ::lib.schema.join/join]::joinable])+(mr/def ::join-clause-source+ [:or ::lib.join.util/partial-join ::lib.schema/query ::lib.schema/stage ::joinable])+(declare with-join-alias)(defn- with-join-alias-update-join-fields@@ -152,17 +156,18 @@(with-join-alias-update-join-fields new-alias)(with-join-alias-update-join-conditions old-alias new-alias))))-(mu/defn with-join-alias :- [:or- [:map- [:lib/type [:enum :metadata/column :mbql/join]]]- [:ref :mbql.clause/field]]+(mr/def ::with-join-alias-target+ "A `:field` ref, a partial join, or a column that may be part-way through getting its join alias added or removed."+ [:multi {:dispatch lib.dispatch/dispatch-value}+ [:field [:ref :mbql.clause/field]]+ [:metadata/column [:ref ::lib.schema.metadata/column.map]]+ [:mbql/join [:ref ::lib.join.util/partial-join]]])++(mu/defn with-join-alias :- ::lib.join.util/column-or-field-ref-or-partial-join"Add OR REMOVE a specific `join-alias` to `field-or-join`, which is either a `:field`/Field metadata, or a join map.Does not recursively update other references (yet; we can add this in the future)."{:style/indent [:form]}- [field-or-join :- [:or- [:map- [:lib/type [:enum :metadata/column :mbql/join]]]- [:ref :mbql.clause/field]]+ [field-or-join :- ::with-join-alias-targetjoin-alias :- [:maybe ::lib.schema.common/non-blank-string]](case (lib.dispatch/dispatch-value field-or-join):field@@ -249,13 +254,11 @@(throw (ex-info "You can't calculate a metadata map for a join! Use lib.metadata.calculation/returned-columns-method instead."{})))-(mu/defn column-from-join :- [:map- [:lib/type [:= :metadata/column]]]+(mu/defn column-from-join :- ::lib.schema.metadata/column"For a column that comes from a join, add or update metadata as needed, e.g. include join name in the display name."[query :- ::lib.schema/querystage-number :- :int- col :- [:map- [:lib/type [:= :metadata/column]]]+ col :- ::lib.schema.metadata/columnjoin-alias :- ::lib.schema.join/alias](-> col(assoc@@ -264,7 +267,10 @@:lib/join-alias join-alias:lib/original-name ((some-fn :lib/original-name :name) col):lib/original-display-name (or (:lib/original-display-name col)- (lib.metadata.calculation/display-name query stage-number (dissoc col :lib/join-alias :lib/original-join-alias))))+ (lib.metadata.calculation/display-name+ query stage-number+ (cond-> (dissoc col :lib/join-alias :lib/original-join-alias)+ (= (:lib/source col) :source/joins) (dissoc :lib/source)))))(set/rename-keys {:lib/expression-name :lib/original-expression-name})(as-> $col (assoc $col :display-name (lib.metadata.calculation/display-name query stage-number $col)))))@@ -309,7 +315,7 @@[query :- ::lib.schema/query_stage-number :- :int{:keys [stages], :as _join} :- ::lib.schema.join/join- options :- [:maybe ::lib.metadata.calculation/returned-columns.options]]+ options :- [:maybe ::lib.metadata.calculation/visible-columns.options]](let [join-query (assoc query :stages stages)](lib.metadata.calculation/returned-columns join-query -1 -1 options)))@@ -327,13 +333,15 @@Note that a join's `:fields` is effectively relative to the parent stage of the join, rather than the join's laststage, since they include `:join-alias`; if you want to resolve these refs, you should use this function as opposedto `returned-columns`."- ([query stage-number join]+ ([query :- ::lib.schema/query+ stage-number :- :int+ join :- ::lib.schema.join/join](join-returned-columns-relative-to-parent-stage query stage-number join nil))([query :- ::lib.schema/querystage-number :- :int{join-alias :alias, :as join} :- ::lib.schema.join/join- options :- [:maybe ::lib.metadata.calculation/returned-columns.options]]+ options :- [:maybe ::lib.metadata.calculation/visible-columns.options]](into [](comp(map lib.field.util/update-keys-for-col-from-previous-stage)@@ -346,10 +354,10 @@(mu/defn join-fields-to-add-to-parent-stage :- [:maybe [:sequential ::lib.metadata.calculation/column-metadata-with-source]]"The resolved `:fields` from a join, which we automatically append to the parent stage's `:fields`."- [query- stage-number+ [query :- ::lib.schema/query+ stage-number :- :int{:keys [fields stages], join-alias :alias, :or {fields :none}, :as join} :- ::lib.schema.join/join- options :- [:maybe ::lib.metadata.calculation/returned-columns.options]]+ options :- [:maybe ::lib.metadata.calculation/visible-columns.options]](when-not (= fields :none)(let [cols (join-returned-columns-relative-to-parent-stage query stage-number join options)cols' (if (= fields :all)@@ -406,7 +414,7 @@[query :- ::lib.schema/querystage-number :- :int{join-alias :alias, :as join} :- ::lib.schema.join/join- options :- [:maybe ::lib.metadata.calculation/returned-columns.options]]+ options :- [:maybe ::lib.metadata.calculation/visible-columns.options]](into [](comp (map lib.field.util/update-keys-for-col-from-previous-stage)(map #(column-from-join query stage-number % join-alias)))@@ -416,7 +424,7 @@"Convenience for calling [[join-visible-columns-relative-to-parent-stage]] on all of the joins in a query stage."[query :- ::lib.schema/querystage-number :- :int- options :- [:maybe ::lib.metadata.calculation/returned-columns.options]]+ options :- [:maybe ::lib.metadata.calculation/visible-columns.options]](into [](mapcat (fn [join](join-visible-columns-relative-to-parent-stage query stage-number join options)))@@ -426,7 +434,7 @@"Convenience for calling [[join-fields-to-add-to-parent-stage]] on all the joins in a query stage."[query :- ::lib.schema/querystage-number :- :int- options :- [:maybe ::lib.metadata.calculation/returned-columns.options]]+ options :- [:maybe ::lib.metadata.calculation/visible-columns.options]](into [](mapcat (fn [join](join-fields-to-add-to-parent-stage query stage-number join options)))@@ -482,11 +490,15 @@(with-join-alias field join-alias)))))conditions)))+(mr/def ::conditions-input+ "Join conditions as given to [[with-join-conditions]]: boolean MBQL expressions or `:lib/external-op` maps."+ [:sequential [:or ::lib.schema.expression/boolean ::lib.schema.common/external-op]])+(mu/defn with-join-conditions :- ::lib.join.util/partial-join"Update the `:conditions` (filters) for a Join clause."{:style/indent [:form]}[a-join :- ::lib.join.util/partial-join- conditions :- [:maybe [:sequential [:or ::lib.schema.expression/boolean ::lib.schema.common/external-op]]]]+ conditions :- [:maybe ::conditions-input]](let [conditions (-> (mapv lib.common/->op-arg conditions)(with-join-conditions-add-alias-to-rhses (lib.join.util/current-join-alias a-join)))](u/assoc-dissoc a-join :conditions (not-empty conditions))))@@ -495,7 +507,9 @@"Update a join (or a function that will return a join) to include `:fields`, either `:all`, `:none`, or a sequence ofreferences."[joinable :- ::lib.join.util/partial-join- fields :- [:maybe [:or [:enum :all :none] [:sequential some?]]]]+ fields :- [:maybe [:or+ [:enum :all :none]+ [:sequential [:or ::lib.schema.ref/ref ::lib.schema.metadata/column]]]]](let [fields (cond(keyword? fields) fields(= fields []) :none@@ -534,7 +548,7 @@For what the join EXPOSES to its outer stage, see [[with-join-fields]]."[a-join :- ::lib.join.util/partial-join- cols :- [:maybe [:sequential some?]]] ; ideally [:sequential ::lib.schema.metadata/column]+ cols :- [:maybe [:sequential ::lib.schema.metadata/column]]](let [first-stage-type (-> a-join :stages first :lib/type)](when-not (= :mbql.stage/mbql first-stage-type)(throw (ex-info "with-join-source-fields requires the join's first stage to be an MBQL stage"@@ -704,7 +718,7 @@(mu/defn joins :- [:maybe ::lib.schema.join/joins]"Get all joins in a specific `stage` of a `query`. If `stage` is unspecified, returns joins in the final stage of thequery."- ([query]+ ([query :- ::lib.schema/query](joins query -1))([query :- ::lib.schema/querystage-number :- :int]@@ -751,7 +765,7 @@(mu/defn available-join-strategies :- [:sequential ::lib.schema.join/strategy.option]"Get available join strategies for the current Database (based on the Database'ssupported [[metabase.driver/features]]) as raw keywords like `:left-join`."- ([query]+ ([query :- ::lib.schema/query](available-join-strategies query -1));; stage number is not currently used, but it is taken as a parameter for consistency with the rest of Lib@@ -766,14 +780,17 @@"Create an MBQL join map from something that can conceptually be joined against. A `Table`? An MBQL or native query? ASaved Question? You should be able to join anything, and this should return a sensible MBQL join map. Uses a left joinby default."- ([joinable]+ ([joinable :- ::join-clause-source](-> (join-clause-method joinable)(u/assoc-default :fields :all)))- ([joinable conditions]+ ([joinable :- ::join-clause-source+ conditions :- [:maybe ::conditions-input]](join-clause joinable conditions lib.schema.join/default-strategy))- ([joinable conditions strategy]+ ([joinable :- ::join-clause-source+ conditions :- [:maybe ::conditions-input]+ strategy :- [:or ::lib.schema.join/strategy ::lib.schema.join/strategy.option]](-> (join-clause joinable)(with-join-conditions conditions)(with-join-strategy strategy))))@@ -785,7 +802,8 @@(mu/defn join :- ::lib.schema/query"Add a join clause to a `query`."- ([query a-join]+ ([query :- ::lib.schema/query+ a-join :- [:or ::lib.join.util/partial-join ::joinable]](join query -1 a-join))([query :- ::lib.schema/query@@ -885,7 +903,10 @@Results will be returned in a 'somewhat smart' order with PKs and FKs returned before other columns.Unlike most other things that return columns, implicitly-joinable columns ARE NOT returned here."- ([query joinable lhs-expression-or-nil rhs-expression-or-nil]+ ([query :- ::lib.schema/query+ joinable :- [:maybe ::join-or-joinable]+ lhs-expression-or-nil :- [:maybe ::lib.schema.expression/expression]+ rhs-expression-or-nil :- [:maybe ::lib.schema.expression/expression]](join-condition-lhs-columns query -1 joinable lhs-expression-or-nil rhs-expression-or-nil))([query :- ::lib.schema/query@@ -932,7 +953,10 @@mark the current column as `:selected` in the metadata/display info.Results will be returned in a 'somewhat smart' order with PKs and FKs returned before other columns."- ([query joinable lhs-expression-or-nil rhs-expression-or-nil]+ ([query :- ::lib.schema/query+ joinable :- ::join-or-joinable+ lhs-expression-or-nil :- [:maybe ::lib.schema.expression/expression]+ rhs-expression-or-nil :- [:maybe ::lib.schema.expression/expression]](join-condition-rhs-columns query -1 joinable lhs-expression-or-nil rhs-expression-or-nil))([query :- ::lib.schema/query@@ -965,7 +989,9 @@"Return a sequence of valid filter clause operators that can be used to build a join condition. In the Query BuilderUI, this can be chosen at any point before or after choosing the LHS and RHS. Invalid options are not currentlyfiltered out based on values of the LHS or RHS, but in the future we can add this -- see #31174."- ([query lhs-expression-or-nil rhs-expression-or-nil]+ ([query :- ::lib.schema/query+ lhs-expression-or-nil :- [:maybe ::lib.schema.expression/expression]+ rhs-expression-or-nil :- [:maybe ::lib.schema.expression/expression]](join-condition-operators query -1 lhs-expression-or-nil rhs-expression-or-nil))([_query :- ::lib.schema/query@@ -1021,15 +1047,18 @@Question, or another query. Suggested conditions will be returned if the source Table has a foreign key to theprimary key of the thing we're joining (see #31175 for more info); otherwise this will return `nil` if no defaultconditions are suggested."- ([query joinable]+ ([query :- ::lib.schema/query+ joinable :- [:maybe ::joinable]](suggested-join-conditions query -1 joinable nil))- ([query stage-number joinable]+ ([query :- ::lib.schema/query+ stage-number :- :int+ joinable :- [:maybe ::joinable]](suggested-join-conditions query stage-number joinable nil))([query :- ::lib.schema/querystage-number :- :int- joinable+ joinable :- [:maybe ::joinable]position :- [:maybe :int]](let [unjoined (if position;; Drop this join and any later ones so they won't be used as suggestions.@@ -1189,10 +1218,13 @@This function needs to be usable while we are in the process of constructing a join in the context of a given stage,but also needs to work for rendering existing joins. Pass a join in for existing joins, or something [[::joinable]]for ones we are currently building."- ([query join-or-joinable]+ ([query :- ::lib.schema/query+ join-or-joinable :- [:maybe ::join-or-joinable]](join-lhs-display-name query join-or-joinable nil))- ([query join-or-joinable condition-lhs-expression-or-nil]+ ([query :- ::lib.schema/query+ join-or-joinable :- [:maybe ::join-or-joinable]+ condition-lhs-expression-or-nil :- [:maybe [:or ::lib.schema.metadata/column :mbql.clause/field]]](join-lhs-display-name query -1 join-or-joinable condition-lhs-expression-or-nil))([query :- ::lib.schema/querydiff --git a/src/metabase/lib/join/util.cljc b/src/metabase/lib/join/util.cljcindex 55d04189c5c..678e7177d64 100644--- a/src/metabase/lib/join/util.cljc+++ b/src/metabase/lib/join/util.cljc@@ -43,7 +43,7 @@(mu/defn current-join-alias :- [:maybe ::lib.schema.join/alias]"Get the current join alias associated with something, if it has one."- [field-or-join]+ [field-or-join :- [:maybe ::column-or-field-ref-or-partial-join]](case (lib.dispatch/dispatch-value field-or-join):field (:join-alias (lib.options/options field-or-join)):metadata/column (:lib/join-alias field-or-join)diff --git a/src/metabase/lib/limit.cljc b/src/metabase/lib/limit.cljcindex 5d550b53def..256edde5148 100644--- a/src/metabase/lib/limit.cljc+++ b/src/metabase/lib/limit.cljc@@ -18,7 +18,8 @@(mu/defn ^:export limit :- ::lib.schema/query"Set the maximum number of rows to be returned by a stage of a query to `n`. If `n` is `nil`, remove the limit."- ([query n]+ ([query :- ::lib.schema/query+ n :- [:maybe pos-int?]](limit query -1 n))([query :- ::lib.schema/querydiff --git a/src/metabase/lib/measure.cljc b/src/metabase/lib/measure.cljcindex 4f7750912a6..7aea6dcc610 100644--- a/src/metabase/lib/measure.cljc+++ b/src/metabase/lib/measure.cljc@@ -114,7 +114,7 @@(mu/defn available-measures :- [:maybe [:sequential {:min 1} ::lib.schema.metadata/measure]]"Get a list of Measures usable as aggregation in `query`. Only Measures with the same`table-id` as the `source-table` of `query` are returned. Measures for joined tables are NOT returned."- ([query]+ ([query :- ::lib.schema/query](available-measures query -1))([query :- ::lib.schema/querystage-number :- :int]diff --git a/src/metabase/lib/metadata.cljc b/src/metabase/lib/metadata.cljcindex f0363cd9106..daef8aa0918 100644--- a/src/metabase/lib/metadata.cljc+++ b/src/metabase/lib/metadata.cljc@@ -82,11 +82,13 @@Options:- `:include-sensitive?` - if true, includes fields with visibility_type :sensitive (default false)"- ([metadata-providerable table-id]+ ([metadata-providerable :- ::lib.schema.metadata/metadata-providerable+ table-id :- ::lib.schema.id/table](active-fields metadata-providerable table-id nil))([metadata-providerable :- ::lib.schema.metadata/metadata-providerabletable-id :- ::lib.schema.id/table- opts]+ opts :- [:maybe [:map {:closed true}+ [:include-sensitive? {:optional true} [:maybe :boolean]]]]](fields* metadata-providerable table-id opts)))(mu/defn metadatas-for-table :- [:maybe [:sequential [:ordiff --git a/src/metabase/lib/metadata/cache.cljc b/src/metabase/lib/metadata/cache.cljcindex fa3a5fc49e9..fda51dd1816 100644--- a/src/metabase/lib/metadata/cache.cljc+++ b/src/metabase/lib/metadata/cache.cljc@@ -4,10 +4,13 @@easy to cache things like `visible-columns`."(:refer-clojure :exclude [not-empty])(:require+ [malli.core :as mc]+ [malli.util :as mut][medley.core :as m][metabase.lib.dispatch :as lib.dispatch][metabase.lib.metadata.protocols :as lib.metadata.protocols][metabase.lib.metadata.util :as lib.metadata.util]+ [metabase.lib.schema :as lib.schema][metabase.lib.util :as lib.util][metabase.util :as u][metabase.util.log :as log]@@ -16,10 +19,46 @@[metabase.util.performance :refer [not-empty]])#?(:cljs (:require-macros [metabase.lib.metadata.cache])))+(mr/def ::options+ "Options accepted as the last positional arg to [[cache-key]]: whatever+ [[metabase.lib.metadata.calculation/returned-columns]]/[[metabase.lib.metadata.calculation/visible-columns]]+ accept, plus the extra keys [[metabase.lib.metadata.calculation/cacheable-options]] adds so caching stays correct+ across the dynamic variables that can affect metadata calculation."+ [:maybe+ [:merge+ [:ref :metabase.lib.metadata.calculation/visible-columns.options]+ [:map {:closed true}+ [:metabase.lib.metadata.calculation/display-name-style {:optional true} :metabase.lib.metadata.calculation/display-name-style]+ [:metabase.lib.metadata.calculation/propagate-binning-and-bucketing {:optional true} :boolean]+ [:metabase.lib.metadata.calculation/ref-style {:optional true} :keyword]]]])++(defn- query-with-hashed-metadata-schema+ "`::lib.schema/query`'s shape, but with `:lib/metadata` replaced by its `hash` (an int) instead of an actual+ metadata provider, as produced by `prepare-map` in [[cache-key-for-other]] and [[cache-key-for-table-or-card]]."+ []+ (let [query-map-schema (first (mc/children (mr/resolve-schema ::lib.schema/query)))]+ (mut/merge query-map-schema [:map {:closed true} [:lib/metadata {:optional true} [:maybe :int]]])))++(mr/def ::query-with-hashed-metadata (query-with-hashed-metadata-schema))++(mr/def ::element+ "A single element of a [[cache-key]]: any of the pieces of data [[cache-key-for-table-or-card]]+ and [[cache-key-for-other]] combine to build one."+ [:or+ :int+ :string+ :keyword+ :boolean+ :nil+ ::lib.schema/query+ ::query-with-hashed-metadata+ :metabase.lib.metadata.calculation/displayable+ ::options])+(mr/def ::cache-key[:catqualified-keyword?- [:+ :any]])+ [:+ ::element]])(defn- cache-key-for-table-or-card"Metadata for a Table, Card, or Metric always comes from the metadata provider and is always unaffected by the@@ -73,15 +112,15 @@(mu/defn cache-key :- ::cache-key"Calculate a cache key to use with [[with-cached-value]]. Prefer the 5 arity, which ensures unserializable keyslike `:lib/metadata` are removed."- ([unique-key x]+ ([unique-key :- qualified-keyword?+ x :- ::element][unique-key x])([unique-key :- qualified-keyword?- query :- [:map- [:lib/type [:= :mbql/query]]]+ query :- ::lib.schema/querystage-number :- :int- x :- :any- options :- :any]+ x :- [:maybe :metabase.lib.metadata.calculation/displayable]+ options :- ::options](if (#{:metadata/table :metadata/card :metadata/metric} (lib.dispatch/dispatch-value x))(cache-key-for-table-or-card unique-key query x options)(cache-key-for-other unique-key query stage-number x options))))@@ -92,10 +131,18 @@(when (lib.metadata.protocols/cached-metadata-provider? metadata-provider)metadata-provider)))+(mr/def ::cached-value+ "A value that can be stored in and retrieved from the cache: whatever [[do-with-cached-value]]'s `thunk` can+ return."+ [:or+ :nil+ :metabase.lib.metadata.calculation/displayable+ [:sequential :metabase.lib.metadata.calculation/displayable]])+(mu/defn- cached-value[metadata-providerable :- ::lib.metadata.protocols/metadata-providerablek :- ::cache-key- not-found]+ not-found :- [:= ::not-found]](if-let [metadata-provider (->cached-metadata-provider metadata-providerable)](lib.metadata.protocols/cached-value metadata-provider k not-found)(do (log/warn "Not a cached-metadata-provider")@@ -104,7 +151,7 @@(mu/defn- cache-value! :- :nil[metadata-providerable :- ::lib.metadata.protocols/metadata-providerablek :- ::cache-key- v]+ v :- ::cached-value](if-let [metadata-provider (->cached-metadata-provider metadata-providerable)](lib.metadata.protocols/cache-value! metadata-provider k v)(log/warn "Not a cached-metadata-provider"))@@ -126,7 +173,7 @@"Impl for [[with-cached-value]]."[metadata-providerable :- ::lib.metadata.protocols/metadata-providerablek :- ::cache-key- thunk :- [:=> [:cat] :any]]+ thunk :- [:=> [:cat] ::cached-value]](log/debug (str (u/colorize :cyan "GET: ") (name (first k)) " " (hash (rest k))))(let [cached-v (cached-value metadata-providerable k ::not-found)](if-not (= cached-v ::not-found)diff --git a/src/metabase/lib/metadata/cached_provider.cljc b/src/metabase/lib/metadata/cached_provider.cljcindex fc8e72e2213..2c325b44e67 100644--- a/src/metabase/lib/metadata/cached_provider.cljc+++ b/src/metabase/lib/metadata/cached_provider.cljc@@ -10,10 +10,15 @@[metabase.util :as u][metabase.util.log :as log][metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr][metabase.util.performance :refer [update-keys get-in #?(:clj doseq)]]))#?(:clj (set! *warn-on-reflection* true))+(mr/def ::cache+ "An atom used to memoize metadata lookups."+ [:fn {:error/message "an atom"} #(instance? #?(:clj clojure.lang.IAtom :cljs cljs.core/IAtom) %)])+(defn- get-in-cache [cache ks](when-some [cached-value (get-in @cache ks)](when-not (= cached-value ::nil)@@ -26,7 +31,7 @@value)))(mu/defn- store-metadata!- [cache+ [cache :- ::cachemetadata-type :- ::lib.schema.metadata/typeid :- pos-int?metadata :- [:multi@@ -126,7 +131,9 @@(into [] (mapcat #(get-in-cache cache (cache-key %))) (sort table-ids))))(mu/defn- metadatas- [cache uncached-provider {metadata-type :lib/type, id-set :id, name-set :name, table-ids :table-ids, :as metadata-spec} :- ::lib.metadata.protocols/metadata-spec]+ [cache :- ::cache+ uncached-provider :- ::lib.metadata.protocols/metadata-provider+ {metadata-type :lib/type, id-set :id, name-set :name, table-ids :table-ids, :as metadata-spec} :- ::lib.metadata.protocols/metadata-spec](cond(or id-set name-set)(metadatas-by-id-or-name cache uncached-provider metadata-spec)diff --git a/src/metabase/lib/metadata/calculation.cljc b/src/metabase/lib/metadata/calculation.cljcindex 7798ca62d0b..0856ebfc6e8 100644--- a/src/metabase/lib/metadata/calculation.cljc+++ b/src/metabase/lib/metadata/calculation.cljc@@ -14,9 +14,14 @@[metabase.lib.options :as lib.options][metabase.lib.ref :as lib.ref][metabase.lib.schema :as lib.schema]+ [metabase.lib.schema.binning :as lib.schema.binning][metabase.lib.schema.common :as lib.schema.common]+ [metabase.lib.schema.drill-thru :as lib.schema.drill-thru][metabase.lib.schema.expression :as lib.schema.expression]+ [metabase.lib.schema.extraction :as lib.schema.extraction]+ [metabase.lib.schema.join :as lib.schema.join][metabase.lib.schema.metadata :as lib.schema.metadata]+ [metabase.lib.schema.order-by :as lib.schema.order-by][metabase.lib.schema.temporal-bucketing :as lib.schema.temporal-bucketing][metabase.lib.types.isa :as lib.types.isa][metabase.lib.util :as lib.util]@@ -55,25 +60,92 @@(lib.dispatch/dispatch-value x)):hierarchy lib.hierarchy/hierarchy)+(mr/def ::card-metadata+ "Card metadata, plus the `:display-name` [[metadata]] adds to it."+ [:merge+ ::lib.schema.metadata/card+ [:map+ [:display-name {:optional true} [:maybe :string]]]])++(mr/def ::metric-metadata+ "Metric metadata, plus the `:display-name` [[metadata]] adds and the `:aggregation-position`+ [[metabase.lib.metric/available-metrics]] adds."+ [:merge+ ::lib.schema.metadata/metric+ [:map+ [:display-name {:optional true} [:maybe :string]]+ [:aggregation-position {:optional true} [:maybe :int]]]])++(mr/def ::column-nameable+ "Something [[column-name]] can calculate a database-friendly name for: an MBQL expression clause or literal, or+ column, metric, or measure metadata."+ [:or+ ::lib.schema.expression/expression+ ::lib.schema.metadata/column+ ::metric-metadata+ ::lib.schema.metadata/measure+ ::lib.schema.metadata/segment])++(mr/def ::displayable+ "Something [[display-name]] can calculate a human-friendly display name for: an MBQL clause or literal, an order-by+ clause, a query, a stage, a join, a join strategy, temporal bucketing, or binning option, resolved binning, an+ aggregation operator, a drill, an extraction, a column group, or column, table, card, metric, measure, or segment+ metadata."+ [:multi {:dispatch (fn [x]+ (cond+ (map? x) (:lib/type x)+ (and (vector? x) (#{:asc :desc} (first x))) ::order-by+ :else ::expression))}+ [:metadata/column ::lib.schema.metadata/column]+ [:metadata/table ::lib.schema.metadata/table]+ [:metadata/card ::card-metadata]+ [:metadata/metric ::metric-metadata]+ [:metadata/measure ::lib.schema.metadata/measure]+ [:metadata/segment ::lib.schema.metadata/segment]+ [:metadata/column-group :metabase.lib.column-group/column-group]+ [:mbql/query ::lib.schema/query]+ [:mbql.stage/mbql ::lib.schema/stage]+ [:mbql.stage/native ::lib.schema/stage]+ [:mbql/join [:ref :metabase.lib.join.util/join-with-optional-alias]]+ [:option/join.strategy ::lib.schema.join/strategy.option]+ [:option/temporal-bucketing ::lib.schema.temporal-bucketing/option]+ [:option/binning ::lib.schema.binning/binning-option]+ [:metabase.lib.binning/binning ::lib.schema.binning/binning.resolved]+ [:operator/aggregation :metabase.lib.aggregation/selected-operator-with-columns]+ [:metabase.lib.drill-thru/drill-thru ::lib.schema.drill-thru/drill-thru]+ [:metabase.lib.extraction/extraction ::lib.schema.extraction/extraction]+ [::order-by ::lib.schema.order-by/order-by]+ [::expression ::lib.schema.expression/expression]])++(defn- clause-options+ "The Lib options of `x` when it is an MBQL clause or a column, the only displayables that carry options."+ [x]+ (when (or (vector? x)+ (= (:lib/type x) :metadata/column))+ (lib.options/options x)))+(mu/defn ^:export display-name :- :string"Calculate a nice human-friendly display name for something. See [[::display-name-style]] for a the difference betweendifferent `style`s."- ([query]+ ([query :- ::lib.schema/query](display-name query query))- ([query x]+ ([query :- ::lib.schema/query+ x :- ::displayable](display-name query -1 x))- ([query stage-number x]+ ([query :- ::lib.schema/query+ stage-number :- :int+ x :- ::displayable](display-name query stage-number x *display-name-style*))([query :- ::lib.schema/querystage-number :- :int- x+ x :- ::displayablestyle :- ::display-name-style](or;; if this is an MBQL clause with `:display-name` in the options map, then use that rather than calculating a name.- ((some-fn :display-name :lib/expression-name) (lib.options/options x))+ ((some-fn :display-name :lib/expression-name) (clause-options x))(lib.util/recover(fn [] (display-name-method query stage-number x style))(fn [e]@@ -83,15 +155,16 @@(mu/defn column-name :- ::lib.schema.common/non-blank-string"Calculate a database-friendly name to use for an expression."- ([query x]+ ([query :- ::lib.schema/query+ x :- ::column-nameable](column-name query -1 x))([query :- ::lib.schema/querystage-number :- :int- x]+ x :- ::column-nameable](or;; if this is an MBQL clause with `:name` in the options map, then use that rather than calculating a name.- (:name (lib.options/options x))+ (:name (clause-options x))(lib.util/recover(fn [] (column-name-method query stage-number x))(fn [e]@@ -168,7 +241,8 @@(mu/defn describe-top-level-key :- [:maybe ::lib.schema.common/non-blank-string]"'top-level' here means the top level of an individual stage. Generate a human-friendly string describing a specificpart of an MBQL stage, or `nil` if that part doesn't exist."- ([query top-level-key]+ ([query :- ::lib.schema/query+ top-level-key :- TopLevelKey](describe-top-level-key query -1 top-level-key))([query :- ::lib.schema/querystage-number :- :int@@ -184,16 +258,27 @@(lib.dispatch/dispatch-value expr)):hierarchy lib.hierarchy/hierarchy)+(mr/def ::type-of-arg+ "Something [[type-of]] can calculate the effective type of: an MBQL expression clause or literal, or column,+ segment, metric, or measure metadata."+ [:or+ ::lib.schema.expression/expression+ ::lib.schema.metadata/column+ ::lib.schema.metadata/segment+ ::lib.schema.metadata/metric+ ::lib.schema.metadata/measure])+(mu/defn type-of :- ::lib.schema.common/base-type"Get the effective type of an MBQL expression."- ([query x]+ ([query :- ::lib.schema/query+ x :- ::type-of-arg](type-of query -1 x))([query :- ::lib.schema/querystage-number :- :int- x]+ x :- ::type-of-arg];; this logic happens here so we don't need to code up every single individual method to handle these special;; cases.- (let [{:keys [temporal-unit], :as options} (lib.options/options x)]+ (let [{:keys [temporal-unit], :as options} (clause-options x)](or;; If the options map includes `:effective-type` we can assume you know what you are doing and that it is;; correct and just return it directly.@@ -280,25 +365,26 @@"Calculate an appropriate `:metadata/*` object for something. What this looks like depends on what we're calculatingmetadata for. If it's a reference or expression of some sort, this should return a single `:metadata/column`map (i.e., something satisfying the `::lib.schema.metadata/column` schema."- ([query]+ ([query :- ::lib.schema/query](metadata query -1 query))- ([query x]+ ([query :- ::lib.schema/query+ x :- ::displayable](metadata query -1 x))([query :- ::lib.schema/querystage-number :- :int- x]+ x :- ::displayable](lib.metadata.cache/with-cached-value query (cache-key ::metadata query stage-number x {})(metadata-method query stage-number x))))(mu/defn describe-query :- ::lib.schema.common/non-blank-string"Convenience for calling [[display-name]] on a query to describe the results of its final stage."- [query]+ [query :- ::lib.schema/query](display-name query query))(mu/defn suggested-name :- [:maybe ::lib.schema.common/non-blank-string]"Name you might want to use for a query when saving an previously-unsaved query. This is the sameas [[describe-query]] except for native queries, where we don't describe anything."- [query]+ [query :- ::lib.schema/query](when-not (= (:lib/type (lib.util/query-stage query -1)) :mbql.stage/native)(lib.util/recover(fn [] (describe-query query))@@ -371,12 +457,13 @@"Given some sort of Cljs object, return a map with the info you'd need to implement UI for it. This is mostly meant topower the Frontend JavaScript UI; in JS, results will be converted to plain JavaScript objects, so avoid returningthings that should remain opaque."- ([query x]+ ([query :- ::lib.schema/query+ x :- ::displayable](display-info query -1 x))([query :- ::lib.schema/querystage-number :- :int- x]+ x :- ::displayable](letfn [(display-info* [x](lib.util/recover(fn [] (display-info-method query stage-number x))@@ -401,7 +488,7 @@implementation and add additional information to it."[query :- ::lib.schema/querystage-number :- :int- x]+ x :- ::displayable](let [x-metadata (metadata query stage-number x)](merge;; TODO -- not 100% convinced the FE should actually have access to `:name`, can't it use `:display-name`@@ -486,7 +573,7 @@(mr/def ::returned-columns.options"Schema for options passed to [[returned-columns]] and [[returned-columns-method]]."[:and- [:map+ [:map {:closed true}[:include-remaps? {:optional true, :default false} :boolean][:include-sensitive-fields? {:optional true, :default false} :boolean]][:fn@@ -519,6 +606,17 @@[query _stage-number stage-number options](returned-columns-method query stage-number (lib.util/query-stage query stage-number) options))+(mr/def ::returned-columns-arg+ "Something [[returned-columns]] can calculate returned columns for: an integer stage number of `query`, the query+ itself, one of its stages, a join, a card, or a table."+ [:or+ :int+ ::lib.schema/query+ ::lib.schema/stage+ ::lib.schema.join/join+ ::lib.schema.metadata/card+ ::lib.schema.metadata/table])+(mu/defn returned-columns :- [:maybe ::returned-columns]"Return a sequence of metadata maps for all the columns expected to be 'returned' at a query, stage of the query, orjoin, and include the `:lib/source` of where they came from. This should only include columns that will be present@@ -538,19 +636,22 @@* `returned-columns` for a stage have source and desired aliases relative to that stage!* `returned-columns` for a Card has the same source and desired aliases you'd see in that Card's `:result-metadata`!!"- ([query]+ ([query :- ::lib.schema/query](returned-columns query -1 query))- ([query x]+ ([query :- ::lib.schema/query+ x :- ::returned-columns-arg](returned-columns query -1 x))- ([query stage-number x]+ ([query :- ::lib.schema/query+ stage-number :- :int+ x :- ::returned-columns-arg](returned-columns query stage-number x nil))([query :- ::lib.schema/querystage-number :- :int- x- options :- [:maybe ::returned-columns.options]]+ x :- ::returned-columns-arg+ options :- [:maybe [:ref ::visible-columns.options]]](binding [*propagate-binning-and-bucketing* true];; minor optimization for caching purposes: only keep the options keys that are actually relevant for;; `returned-columns` purposes. As a bonus, this means undocumented options keys that aren't part of the schema@@ -616,10 +717,11 @@Columns from joins, expressions, and implicitly joinable columns are included automatically by default;see `::visible-columns.options` for the options for disabling these columns."- ([query]+ ([query :- [:maybe ::lib.schema/query]](visible-columns query -1))- ([query stage-number]+ ([query :- [:maybe ::lib.schema/query]+ stage-number :- [:maybe :int]];; apparently the FE sometimes accidentally calls this with a `nil` stage number -- see #31366 -- in that case just;; return an empty vector. The FE only has access to this arity, so we don't need to do the check below.;; See [[metabase.lib.metadata.calculation-test/visible-columns-test-2]]@@ -646,7 +748,7 @@[query :- ::lib.schema/querystage-number :- :intsource-cols :- [:maybe [:sequential ::lib.schema.metadata/column]]- {:keys [include-remaps?] :as _options} :- [:maybe ::returned-columns.options]]+ {:keys [include-remaps?] :as _options} :- [:maybe [:ref ::visible-columns.options]]](when (and include-remaps?(lib.util/first-stage? query stage-number))(let [existing-ids (into #{} (keep :id) source-cols)]@@ -739,14 +841,14 @@"If this query has an MBQL first stage with a `:source-table` ID, return the `:metadata/table` for it.Returns nil if the query is native or has a `:source-card`."- [query]+ [query :- ::lib.schema/query](some->> query lib.util/source-table-id (lib.metadata/table query)))(mu/defn primary-source-card :- [:maybe ::lib.schema.metadata/card]"If this query has an MBQL first stage with a `:source-card` ID, return the `:metadata/card` for it.Returns nil if the query is native or has a `:source-table`."- [query]+ [query :- ::lib.schema/query](some->> query lib.util/source-card-id (lib.metadata/card query)))(mu/defn primary-source :- [:maybe [:or ::lib.schema.metadata/card ::lib.schema.metadata/table]]@@ -754,6 +856,6 @@`:metadata/table` or `:metadata/card`.Returns nil if the query is native."- [query]+ [query :- ::lib.schema/query](or (primary-source-table query)(primary-source-card query)))diff --git a/src/metabase/lib/metadata/column.cljc b/src/metabase/lib/metadata/column.cljcindex 169a2ffd651..6c2e98a69ea 100644--- a/src/metabase/lib/metadata/column.cljc+++ b/src/metabase/lib/metadata/column.cljc@@ -26,7 +26,8 @@(mu/defn column-with-unique-key :- [:maybe ::lib.metadata.calculation/returned-column]"Get metadata for the returned column with `unique-key`."- ([query unique-key]+ ([query :- ::lib.schema/query+ unique-key :- ::lib.schema/column-unique-key](column-with-unique-key query -1 unique-key))([query :- ::lib.schema/querystage-number :- :intdiff --git a/src/metabase/lib/metadata/protocols.cljc b/src/metabase/lib/metadata/protocols.cljcindex cdd1f25a3e1..0e30b3e3c2b 100644--- a/src/metabase/lib/metadata/protocols.cljc+++ b/src/metabase/lib/metadata/protocols.cljc@@ -179,9 +179,15 @@#'metadata-providerable?])(mr/def ::metadata- [:map- [:lib/type ::metadata-type-excluding-database]- [:id pos-int?]])+ [:multi {:dispatch :lib/type}+ [:metadata/table [:ref ::lib.schema.metadata/table]]+ [:metadata/column [:ref ::lib.schema.metadata/column]]+ [:metadata/card [:ref ::lib.schema.metadata/card]]+ [:metadata/measure [:ref ::lib.schema.metadata/measure]]+ [:metadata/metric [:ref ::lib.schema.metadata/metric]]+ [:metadata/segment [:ref ::lib.schema.metadata/segment]]+ [:metadata/native-query-snippet [:ref ::lib.schema.metadata/native-query-snippet]]+ [:metadata/transform [:ref ::lib.schema.metadata/transform]]])(mu/defn- metadata-by-id :- [:maybe ::metadata][metadata-provider :- ::metadata-provider@@ -302,7 +308,7 @@(fields metadata-provider table-id nil))([metadata-provider :- ::metadata-providertable-id :- ::lib.schema.id/table- opts :- [:maybe [:map [:include-sensitive? {:optional true} :boolean]]]]+ opts :- [:maybe [:map {:closed true} [:include-sensitive? {:optional true} :boolean]]]];; only include `:include-sensitive?` when truthy, so that this shares cached-provider cache entries with other;; ways of fetching the fields for a table, e.g. [[metadatas-for-tables]](metadatas metadata-provider (cond-> {:lib/type :metadata/column, :table-ids #{table-id}}diff --git a/src/metabase/lib/metadata/result_metadata.cljc b/src/metabase/lib/metadata/result_metadata.cljcindex f9d777cc74f..c688c6ea817 100644--- a/src/metabase/lib/metadata/result_metadata.cljc+++ b/src/metabase/lib/metadata/result_metadata.cljc@@ -48,9 +48,7 @@;; we can 'ban' stuff like `:source-alias` and `:source` within Lib itself. See #59772 for some experimental work;; there. (See QUE2-361)[:and- [:map- [::source {:optional true} ::lib.schema.metadata/column.legacy-source]- [::field-ref {:optional true} ::mbql.s/Reference]]+ ::lib.schema.metadata/column.map(lib.schema.common/disallowed-keys{:source "Use ::source instead of :source":field-ref "Use ::field-ref instead of :field-ref"@@ -70,10 +68,25 @@(mr/def ::cols[:maybe [:sequential ::col]])+(mr/def ::driver-col+ "A [[mbql.s/driver-column]] with its keys kebab-cased."+ [:map {:closed true}+ [:name {:optional true} :string]+ [:base-type {:optional true} [:maybe ::lib.schema.common/base-type]]+ [:effective-type {:optional true} [:maybe ::lib.schema.common/base-type]]+ [:semantic-type {:optional true} [:maybe ::lib.schema.common/semantic-or-relation-type]]+ [:database-type {:optional true} [:maybe :string]]+ [:display-name {:optional true} [:maybe :string]]+ [:field-ref {:optional true} [:maybe [:ref ::mbql.s/Reference]]]])++(mr/def ::initial-cols+ "The columns a driver reported, as the QP passes them in: legacy result metadata or bare driver columns."+ [:maybe [:sequential [:or ::mbql.s/legacy-column-metadata ::mbql.s/driver-column ::driver-col]]])+(mu/defn- merge-col :- ::col"Merge a map from `:cols` returned by the driver with the column metadata from Lib. We'll generally prefer the valuesfrom the driver to values calculated by Lib."- [driver-col :- [:maybe ::col]+ [driver-col :- [:maybe [:or ::col ::driver-col]]lib-col :- [:maybe ::col]](let [driver-col (update-keys driver-col u/->kebab-case-en)driver-base-type (:base-type driver-col)@@ -106,7 +119,7 @@It's the responsibility of the driver to make sure the `:cols` are returned in the correct number and order (matchingthe order supposed by Lib)."- [initial-cols :- [:maybe [:sequential ::kebab-cased-map]]+ [initial-cols :- [:maybe [:sequential [:or ::kebab-cased-map ::driver-col]]]lib-cols :- [:maybe [:sequential ::kebab-cased-map]]](cond(= (count initial-cols) (count lib-cols))@@ -154,14 +167,15 @@(mu/defn- basic-native-col :- ::kebab-cased-map"Generate basic column metadata for a column coming back from a native query for which we have only barebones metadatacoming back from the driver like name and base type."- [col :- ::col]+ [col :- [:or ::mbql.s/legacy-column-metadata ::mbql.s/driver-column ::driver-col]](let [base-type (or ((some-fn :base-type :base_type) col):type/*)]- {:lib/type :metadata/column- :lib/source :source/native- :display-name (:name col)- :base-type base-type- :effective-type base-type}))+ (cond-> {:lib/type :metadata/column+ :lib/source :source/native+ :display-name (:name col)+ :base-type base-type+ :effective-type base-type}+ (:name col) (assoc :name (:name col)))));;; TODO (Cam 6/17/25) -- move this into [[metabase.lib.expression]] or something and have it be part of the mainline;;; methods for metadata calculation@@ -435,7 +449,7 @@(mu/defn- add-extra-metadata :- [:sequential ::kebab-cased-map]"Add extra metadata to the [[lib/returned-columns]] that only comes back with QP results metadata."[query :- ::lib.schema/query- initial-cols :- ::cols]+ initial-cols :- ::initial-cols](binding [lib.metadata.calculation/*display-name-style* :long](let [lib-cols (doall (lib.metadata.calculation/returned-columnsquery@@ -525,11 +539,11 @@name and base type). If provided these are merged with the columns the query is expected to return.Note this `initial-cols` is more or less required for native queries unless they have metadata attached."- ([query]+ ([query :- ::lib.schema/query](returned-columns query []))([query :- ::lib.schema/query- initial-cols :- ::cols]+ initial-cols :- ::initial-cols](->> initial-cols(add-extra-metadata query)cols->legacy-metadata)))diff --git a/src/metabase/lib/metadata/util.cljc b/src/metabase/lib/metadata/util.cljcindex bdafa96fea6..3f9be68eea7 100644--- a/src/metabase/lib/metadata/util.cljc+++ b/src/metabase/lib/metadata/util.cljc@@ -8,7 +8,7 @@(mu/defn ->metadata-provider :- ::lib.schema.metadata/metadata-provider"Get a MetadataProvider from something that can provide one."- ([metadata-providerable]+ ([metadata-providerable :- ::lib.schema.metadata/metadata-providerable](->metadata-provider metadata-providerable nil))([metadata-providerable :- ::lib.schema.metadata/metadata-providerablediff --git a/src/metabase/lib/metric.cljc b/src/metabase/lib/metric.cljcindex 25c979e958a..45aece2bba2 100644--- a/src/metabase/lib/metric.cljc+++ b/src/metabase/lib/metric.cljc@@ -103,7 +103,7 @@(mu/defn available-metrics :- [:maybe [:sequential {:min 1} ::lib.schema.metadata/metric]]"Get a list of Metrics that you may consider using as aggregations for a query."- ([query]+ ([query :- ::lib.schema/query](available-metrics query -1))([query :- ::lib.schema/querystage-number :- :int]diff --git a/src/metabase/lib/native.cljc b/src/metabase/lib/native.cljcindex 53b41e100d5..3e39298fd23 100644--- a/src/metabase/lib/native.cljc+++ b/src/metabase/lib/native.cljc@@ -34,9 +34,13 @@(mr/def ::incomplete-template-tag"An unfinished template tag; must be completed by [[finish-tag]]."- [:map- [:type ::lib.schema.template-tag/type]- [:name ::lib.schema.template-tag/name]])+ [:or+ [:ref ::lib.schema.template-tag/template-tag]+ [:map {:closed true}+ [:type ::lib.schema.template-tag/type]+ [:name ::lib.schema.template-tag/name]+ [:id {:optional true} [:ref ::lib.schema.template-tag/id]]+ [:display-name {:optional true} ::common/non-blank-string]]])(mr/def ::incomplete-template-tags[:sequential ::incomplete-template-tag])@@ -215,7 +219,7 @@#{:collection})(mr/def ::native-extras- [:map+ [:map {:closed true}[:collection {:optional true} ::common/non-blank-string]])(mu/defn required-native-extras :- set?@@ -259,10 +263,10 @@native-extras :- [:maybe ::native-extras]](let [tags (extract-template-tags metadata-providerable sql-or-other-native-query)](cond-> (lib.query/query-with-stages metadata-providerable- [{:lib/type :mbql.stage/native- :lib/stage-metadata results-metadata- :template-tags tags- :native sql-or-other-native-query}])+ [(m/assoc-some {:lib/type :mbql.stage/native+ :template-tags tags+ :native sql-or-other-native-query}+ :lib/stage-metadata results-metadata)])native-extras (with-native-extras native-extras)))))(mu/defn with-different-database :- ::lib.schema/query@@ -422,7 +426,8 @@(mu/defn- validate-template-tag :- [:sequential [:map [:error/message :string] [:tag-name :string]]]"Validate a single template tag, returning a list of errors."- [_query {tag-type :type tag-name :name, :keys [display-name dimension table-id]}]+ [_query :- ::lib.schema/query+ {tag-type :type tag-name :name, :keys [display-name dimension table-id]} :- ::lib.schema.template-tag/template-tag](cond-> [](empty? display-name)(conj {:error/message (i18n/tru "Missing widget label: {0}" tag-name)@@ -438,7 +443,7 @@(mu/defn validate-template-tags :- [:sequential [:map [:error/message :string] [:tag-name :string]]]"Given a query, returns a list of errors for each template tag in the query that is not valid."- [query]+ [query :- ::lib.schema/query](mapcat #(validate-template-tag query %)(lib.walk.util/all-template-tags query)))@@ -566,7 +571,7 @@(mu/defn native-query-table-references :- [:set [:map [:table ::lib.schema.id/table]]]"Given a native query, find any table tags and convert them to {:table id} objects"- [query]+ [query :- ::lib.schema/query](let [tags (->> (lib.walk.util/all-template-tags query)(filter #(= (:type %) :table)))](into #{}diff --git a/src/metabase/lib/normalize.cljc b/src/metabase/lib/normalize.cljcindex f267beaad6d..7d816047bff 100644--- a/src/metabase/lib/normalize.cljc+++ b/src/metabase/lib/normalize.cljc@@ -106,7 +106,7 @@"Take a sequence of legacy or Lib metadata maps, convert to Lib-style if needed, then normalize them.Note that this returns a map with `:columns` rather than a sequence of columns like the input."- [cols :- [:sequential :map]]+ [cols :- [:sequential ::lib.schema.metadata/lib-or-legacy-column]](->> colslib.util/->stage-metadata(normalize ::lib.schema.metadata/stage)))diff --git a/src/metabase/lib/options.cljc b/src/metabase/lib/options.cljcindex 2db902a3290..4f13728b3f5 100644--- a/src/metabase/lib/options.cljc+++ b/src/metabase/lib/options.cljc@@ -5,12 +5,31 @@[metabase.util :as u][metabase.util.i18n :as i18n][metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr][metabase.util.performance :refer [not-empty]]))(defn- mbql-clause? [x](and (vector? x)(keyword? (first x))))+(mr/def ::clause-or-column+ [:multi {:dispatch (fn [x]+ (cond+ (map? x) (:lib/type x ::legacy-query)+ (sequential? x) ::clause+ :else ::clause-arg))}+ [:metadata/column :metabase.lib.schema.metadata/column]+ [:metadata/table :metabase.lib.schema.metadata/table]+ [:metadata/card :metabase.lib.schema.metadata/card]+ [:metadata/metric :metabase.lib.schema.metadata/metric]+ [:mbql/query :metabase.lib.schema/query]+ [:mbql.stage/mbql :metabase.lib.schema/stage.mbql]+ [:mbql.stage/native :metabase.lib.schema/stage.native]+ [:mbql/join [:ref :metabase.lib.join.util/join-with-optional-alias]]+ [::legacy-query :metabase.legacy-mbql.schema/Query]+ [::clause ::lib.schema.common/any-clause]+ [::clause-arg ::lib.schema.common/clause-arg]])+(mu/defn options :- [:maybe map?]"Return the Metabase lib options map associated with an `x`. Lib options is currently used mostly forthe `:lib/uuid` we attach to everything to facilitate removing or replacing clauses later, but we will probably@@ -28,7 +47,7 @@Maybe options should be included directly in the map, but then we'd have to decide which keys are and are notoptions. Is a join `:alias` an option? Probably. What about a `:condition`? It's not optional. So for purposes ofwriting Metabase lib and tracking `:lib/uuid`, this approach seems ok in the short term."- [x]+ [x :- ::clause-or-column](cond(map? x)(:lib/options x)@@ -49,7 +68,8 @@You should probably prefer [[update-options]] to using this directly, so you don't stomp over existing stuffunintentionally. Implement this if you need to teach Metabase lib how to support something that doesn't follow theusual patterns described in [[options]]."- [x new-options :- [:maybe map?]]+ [x :- ::clause-or-column+ new-options :- [:maybe ::lib.schema.common/clause-options]](cond(map? x)(u/assoc-dissoc x :lib/options (not-empty new-options))@@ -83,5 +103,5 @@(mu/defn uuid :- [:maybe ::lib.schema.common/non-blank-string]"Get the `:lib/uuid` associated with something, e.g. an MBQL clause or join."- [x]+ [x :- ::clause-or-column](:lib/uuid (options x)))diff --git a/src/metabase/lib/order_by.cljc b/src/metabase/lib/order_by.cljcindex 1629de8b718..570e15030c5 100644--- a/src/metabase/lib/order_by.cljc+++ b/src/metabase/lib/order_by.cljc@@ -67,10 +67,10 @@(mu/defn order-by-clause"Create an order-by clause independently of a query, e.g. for `replace` or whatever."- ([orderable]+ ([orderable :- [:or ::lib.schema.order-by/order-by ::lib.ref/referenceable]](order-by-clause orderable :asc))- ([orderable :- some?+ ([orderable :- [:or ::lib.schema.order-by/order-by ::lib.ref/referenceable]direction :- [:maybe [:enum :asc :desc]]](-> (order-by-clause-method orderable)(with-direction (or direction :asc)))))@@ -86,15 +86,18 @@You can teach Metabase lib how to generate order by clauses for different things by implementing theunderlying [[order-by-clause-method]] multimethod."- ([query orderable]+ ([query :- ::lib.schema/query+ orderable :- [:or ::lib.schema.order-by/order-by ::lib.ref/referenceable]](order-by query -1 orderable nil))- ([query orderable direction]+ ([query :- ::lib.schema/query+ orderable :- [:or ::lib.schema.order-by/order-by ::lib.ref/referenceable]+ direction :- [:maybe [:enum :asc :desc]]](order-by query -1 orderable direction))- ([query+ ([query :- ::lib.schema/querystage-number :- [:maybe :int]- orderable :- some?+ orderable :- [:or ::lib.schema.order-by/order-by ::lib.ref/referenceable]direction :- [:maybe [:enum :asc :desc]]](let [stage-number (or stage-number -1)new-order-by (cond-> (order-by-clause-method orderable)@@ -195,7 +198,7 @@(mu/defn remove-all-order-bys :- ::lib.schema/query"Remove all order bys from this stage of the query."- ([query]+ ([query :- ::lib.schema/query](remove-all-order-bys query -1))([query :- ::lib.schema/querydiff --git a/src/metabase/lib/page.cljc b/src/metabase/lib/page.cljcindex 05c49282c39..58537144fc0 100644--- a/src/metabase/lib/page.cljc+++ b/src/metabase/lib/page.cljc@@ -6,7 +6,7 @@(mu/defn current-page :- [:maybe ::lib.schema/page]"Return the `:page` in a query stage."- ([query]+ ([query :- ::lib.schema/query](current-page query -1))([query :- ::lib.schema/query@@ -15,7 +15,8 @@(mu/defn with-page :- ::lib.schema/query"Set or remove the `:page` in a query stage."- ([query page]+ ([query :- ::lib.schema/query+ page :- [:maybe ::lib.schema/page]](with-page query -1 page))([query :- ::lib.schema/querydiff --git a/src/metabase/lib/parameters.cljc b/src/metabase/lib/parameters.cljcindex 0f5883809c2..8838c183e1a 100644--- a/src/metabase/lib/parameters.cljc+++ b/src/metabase/lib/parameters.cljc@@ -118,12 +118,12 @@0))(mu/defn update-parameter-target-field-ref :- ::lib.schema.parameter/target- "If parameter `:target` wraps a legacy `:field` ref, convert it to MBQL 5, apply `(apply f field-ref args)`, convert- the result back to legacy, and place it back in the target. If there is no `:field` ref, returns target unchanged."+ "If parameter `:target` wraps a legacy `:field` ref, convert it to MBQL 5, apply `(f field-ref)`, convert the result+ back to legacy, and place it back in the target. If there is no `:field` ref, returns target unchanged."[target :- ::lib.schema.parameter/target- f & args]+ f :- ifn?](if-let [field-ref (parameter-target-field-ref target)]- (let [new-legacy-ref (-> (apply f field-ref args)+ (let [new-legacy-ref (-> (f field-ref)lib.convert/->legacy-MBQL)](case (first target):dimension (let [[tag _ref opts] target]@@ -139,8 +139,8 @@"If parameter `:target` is a `:dimension` pseudo-clause, update the options map associated with it, if any. If it isnot a `:dimension` psuedo-clause, this function no-ops."[target :- ::lib.schema.parameter/target- f & args]+ f :- ifn?](case (first target):dimension (let [[_tag dimension opts] target]- [:dimension dimension (apply f opts args)])+ [:dimension dimension (f opts)])target))diff --git a/src/metabase/lib/parameters/parse/types.cljc b/src/metabase/lib/parameters/parse/types.cljcindex e30b4e87ba3..29a297d321c 100644--- a/src/metabase/lib/parameters/parse/types.cljc+++ b/src/metabase/lib/parameters/parse/types.cljc@@ -2,6 +2,7 @@"Schemas and helper functions for various PARSED parameter maps. These are created from the `:parameters` passed inwith a query."(:require+ [metabase.lib.schema.common :as lib.schema.common][metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.literal :as lib.schema.literal][metabase.lib.schema.metadata :as lib.schema.metadata]@@ -20,9 +21,8 @@[:= ::no-value])(mr/def ::field-filter.value.map- [:map- [:type ::lib.schema.parameter/type]- [:value :any]])+ "The query parameter a field filter's value comes from."+ [:ref ::lib.schema.parameter/parameter])(mr/def ::field-filter.value[:or@@ -47,7 +47,7 @@* A vector of maps like the one above (for multiple values)* Alias is optional and added by #61118 (not sure what it does, look at PR for more info)"- [:map+ [:map {:closed true}[:lib/type [:= ::field-filter]][:field ::lib.schema.metadata/column][:value ::field-filter.value]@@ -60,7 +60,9 @@{:lib/type ::field-filter:field field:value value})- ([field value param-alias]+ ([field :- ::lib.schema.metadata/column+ value :- ::field-filter.value+ param-alias :- [:maybe :string]](assoc (field-filter field value) :alias param-alias)))(defn field-filter?@@ -68,22 +70,27 @@[x](= (:lib/type x) ::field-filter))+(mr/def ::temporal-unit.value+ "Either the no-value placeholder, or the name of a temporal unit as a string."+ [:or ::no-value :string])+(mr/def ::temporal-unit- [:map+ [:map {:closed true}[:lib/type [:= ::temporal-unit]][:field ::lib.schema.metadata/column]- ;; TODO (Cam 7/16/25) -- constrain `:value`- [:value some?]+ [:value ::temporal-unit.value][:alias {:optional true} [:maybe string?]]])(mu/defn temporal-unit :- ::temporal-unit"Create a parsed `temporal-unit` parameter from map `m`."- ([column :- ::lib.schema.metadata/column- value :- some?]+ ([column :- ::lib.schema.metadata/column+ value :- ::temporal-unit.value]{:lib/type ::temporal-unit:field column:value value})- ([column value param-alias]+ ([column :- ::lib.schema.metadata/column+ value :- ::temporal-unit.value+ param-alias :- [:maybe :string]](assoc (temporal-unit column value) :alias param-alias)))(defn temporal-unit?@@ -100,19 +107,20 @@`parameters` are positional parameters for a parameterized native query e.g. the JDBC parameters corresponding to`?` placeholders"- [:map+ [:map {:closed true}[:lib/type [:= ::referenced-card-query]][:card-id ::lib.schema.id/card][:query :string]- [:parameters {:optional true} [:maybe [:sequential :any]]]])+ [:parameters {:optional true} [:maybe [:sequential ::lib.schema.literal/param-value]]]])(mu/defn referenced-card-query :- ::referenced-card-query"Create a parsed `referenced-card-query` parameter from map `m`."- ([card-id query]+ ([card-id :- ::lib.schema.id/card+ query :- :string](referenced-card-query card-id query nil))([card-id :- ::lib.schema.id/cardquery :- :string- parameters :- [:maybe [:sequential :any]]]+ parameters :- [:maybe [:sequential ::lib.schema.literal/param-value]]]{:lib/type ::referenced-card-query:card-id card-id:query query@@ -129,10 +137,10 @@* `:field-id` - the ID of the field to filter on* `:op` - the comparison operator, one of :>, :>=, :<, :<=, :=, :!=* `:value` - the value to compare against"- [:map+ [:map {:closed true}[:field-id ::lib.schema.id/field][:op [:enum :> :>= :< :<= := :!=]]- [:value any?]])+ [:value ::lib.schema.parameter/parameter.value]])(mr/def ::referenced-table-query"`table-id` is the id of the table being referenced@@ -149,7 +157,7 @@\"table\" AS \"alias\" or (SELECT ...) AS \"alias\".Resolved from the template tag's `:emit-alias` boolean and `:name` during parsing."- [:map+ [:map {:closed true}[:lib/type [:= ::referenced-table-query]][:table-id ::lib.schema.id/table][:source-filters {:optional true} [:maybe [:sequential ::referenced-table-query.source-filter]]]@@ -157,9 +165,10 @@(mu/defn referenced-table-query :- ::referenced-table-query"Create a parsed `referenced-table-query` parameter from map `m`."- ([table-id]+ ([table-id :- ::lib.schema.id/table](referenced-table-query table-id nil))- ([table-id source-filters]+ ([table-id :- ::lib.schema.id/table+ source-filters :- [:maybe [:sequential ::referenced-table-query.source-filter]]](referenced-table-query table-id source-filters nil))([table-id :- ::lib.schema.id/tablesource-filters :- [:maybe [:sequential ::referenced-table-query.source-filter]]@@ -181,7 +190,7 @@`snippet-id` is the integer ID of the row in the application DB from where the snippet content is loaded.`content` is the raw query snippet which will be replaced, verbatim, for this template tag."- [:map+ [:map {:closed true}[:lib/type [:= ::referenced-query-snippet]][:snippet-id ::lib.schema.id/snippet][:content :string]])@@ -207,7 +216,7 @@;; TODO (Cam 2026-05-14) -- rename to `::datetime`(mr/def ::date"As in a literal date, defined by date-string `s`."- [:map+ [:map {:closed true}[:lib/type [:= ::date]][:s ::date.value]])@@ -222,15 +231,15 @@(= (:lib/type x) ::date))(mr/def ::date-range- [:map+ [:map {:closed true}[:lib/type [:= ::date-range]]- ;; TODO (Cam 7/16/25) -- constrain `:start` and `:end` values- [:start :any]- [:end :any]])+ [:start [:maybe ::lib.schema.common/non-blank-string]]+ [:end [:maybe ::lib.schema.common/non-blank-string]]])(mu/defn date-range :- ::date-range"Create a new parsed `date-range` parameter from map `m`."- [start end]+ [start :- [:maybe ::lib.schema.common/non-blank-string]+ end :- [:maybe ::lib.schema.common/non-blank-string]]{:lib/type ::date-range, :start start, :end end})(defn date-range?@@ -239,15 +248,15 @@(= (:lib/type x) ::date-range))(mr/def ::date-time-range- [:map+ [:map {:closed true}[:lib/type [:= ::date-time-range]]- ;; TODO (Cam 7/16/25) -- constrain `:start` and `:end` values- [:start :any]- [:end :any]])+ [:start [:maybe ::lib.schema.common/non-blank-string]]+ [:end [:maybe ::lib.schema.common/non-blank-string]]])(mu/defn date-time-range :- ::date-time-range"Create a new parsed `date-time-range` parameter from map `m`."- [start end]+ [start :- [:maybe ::lib.schema.common/non-blank-string]+ end :- [:maybe ::lib.schema.common/non-blank-string]]{:lib/type ::date-time-range, :start start, :end end})(defn date-time-range?@@ -256,7 +265,7 @@(= (:lib/type x) ::date-time-range))(mr/def ::param- [:map+ [:map {:closed true}[:lib/type [:= ::param]][:k :string]])@@ -272,15 +281,16 @@(= (:lib/type x) ::param))(mr/def ::function-param- [:map+ [:map {:closed true}[:lib/type [:= ::function-param]][:function-name :string]- ;; TODO (Cam 7/16/25) -- constrain further; I think these have to be valid parameters- [:args [:sequential :any]]])+ [:args [:sequential [:ref :metabase.lib.parameters.parse/parsed-token]]]])(mu/defn function-param :- ::function-param"Create a new parsed `function-param` from map `m`."- [m :- :map]+ [m :- [:map {:closed true}+ [:function-name :string]+ [:args [:sequential [:ref :metabase.lib.parameters.parse/parsed-token]]]]](assoc m :lib/type ::function-param))(defn function-param?@@ -289,14 +299,13 @@(= (:lib/type x) ::function-param))(mr/def ::optional- [:map+ [:map {:closed true}[:lib/type [:= ::optional]]- ;; TODO (Cam 7/16/25) -- constrain further- [:args [:sequential :any]]])+ [:args [:sequential [:ref :metabase.lib.parameters.parse/parsed-token]]]])(mu/defn optional :- ::optional"Create a new parsed `optional` param from map `m`."- [args :- [:sequential :any]]+ [args :- [:sequential [:ref :metabase.lib.parameters.parse/parsed-token]]]{:lib/type ::optional:args args})@@ -304,3 +313,18 @@"Whether `x` is a map that represents a parsed [[optional]] parameter."[x](= (:lib/type x) ::optional))++(mr/def ::parsed-value+ "The value looked up for a parameter name in a `param->value` map during native query parameter substitution: one+ of the parsed parameter types above, the no-value placeholder, or a literal value (or sequence of literal values)+ for parameters that aren't Field filters, temporal units, or references."+ [:or+ ::no-value+ ::field-filter+ ::temporal-unit+ ::date+ ::referenced-card-query+ ::referenced-table-query+ ::referenced-query-snippet+ ::lib.schema.literal/param-value+ [:sequential ::lib.schema.literal/param-value]])diff --git a/src/metabase/lib/query.cljc b/src/metabase/lib/query.cljcindex 0679f825b2b..1833de7b144 100644--- a/src/metabase/lib/query.cljc+++ b/src/metabase/lib/query.cljc@@ -117,7 +117,8 @@(mu/defn add-types-to-fields"Add `:base-type` and `:effective-type` to options of fields in `x` using `metadata-provider`. Works on MBQL 5 fields.`:effective-type` is required for coerced fields to pass schema checks."- [x metadata-provider :- ::lib.schema.metadata/metadata-provider]+ [x :- [:or ::lib.schema/query ::lib.schema/stage]+ metadata-provider :- ::lib.schema.metadata/metadata-provider](if-let [field-ids (match/match-many x[:field(opts :guard (and (map? opts) (not (and (:base-type opts) (:effective-type opts)))))@@ -150,12 +151,13 @@(mu/defn query-with-stages :- ::lib.schema/query"Create a query from a sequence of stages."- ([metadata-providerable stages]+ ([metadata-providerable :- ::lib.schema.metadata/metadata-providerable+ stages :- [:sequential ::lib.util/query-like]](query-with-stages (:id (lib.metadata/database metadata-providerable)) metadata-providerable stages))([database-id :- ::lib.schema.id/databasemetadata-providerable :- ::lib.schema.metadata/metadata-providerable- stages]+ stages :- [:sequential ::lib.util/query-like]](->> (merge{:lib/type :mbql/query:lib/metadata (lib.metadata/->metadata-provider metadata-providerable)@@ -173,9 +175,11 @@(let [mbql5-query (binding [lib.schema.expression/*suppress-expression-type-check?* true](lib.convert/->mbql5 (mbql.normalize/normalize-or-throw legacy-query)))mp (lib.metadata/->metadata-provider metadata-providerable (:database mbql5-query))- mbql5-query (add-types-to-fields mbql5-query mp)]+ converted? (:lib.convert/converted? mbql5-query)+ mbql5-query (-> mbql5-query (dissoc :lib.convert/converted?) (add-types-to-fields mp))](merge- mbql5-query+ (cond-> mbql5-query+ converted? (assoc :lib.convert/converted? true))(query-with-stages mp (:stages mbql5-query)))))(fn [e](throw (ex-info (i18n/tru "Error creating query from legacy query: {0}" (ex-message e))@@ -186,7 +190,9 @@"Implementation for [[query]]."{:arglists '([metadata-providerable x])}(fn [_metadata-providerable x]- ((some-fn lib.util/normalized-query-type lib.dispatch/dispatch-value) x))+ (or (when (and (map? x) (some #(contains? x %) [:lib/type :type "lib/type" "type"]))+ (lib.util/normalized-query-type x))+ (lib.dispatch/dispatch-value x))):hierarchy lib.hierarchy/hierarchy)(defmethod query-method :query ; legacy MBQL query@@ -299,7 +305,7 @@existing MBQL query or saved question or whatever. If the thing in question does not already include metadata, passit in separately -- metadata is needed for most query manipulation operations."[metadata-providerable :- ::lib.schema.metadata/metadata-providerable- x :- some?]+ x :- ::lib.util/query-like](ensure-cached-metadata-provider (query-method metadata-providerable x)))(mu/defn ->query :- ::lib.schema/query@@ -308,7 +314,7 @@Create a new MBQL query from anything that could conceptually be an MBQL query, like a Database or Table or anexisting MBQL query or saved question or whatever. If the thing in question does not already include metadata, passit in separately -- metadata is needed for most query manipulation operations."- [x+ [x :- ::lib.util/query-likemetadata-providerable :- ::lib.schema.metadata/metadata-providerable](query metadata-providerable x))@@ -316,7 +322,7 @@"Create a MBQL 5 query from a legacy inner query."[metadata-providerable :- ::lib.schema.metadata/metadata-providerabledatabase-id :- ::lib.schema.id/database- inner-query :- :map]+ inner-query :- :metabase.legacy-mbql.schema/SourceQuery](->> (lib.convert/legacy-query-from-inner-query database-id inner-query)lib.convert/->mbql5(query metadata-providerable)))diff --git a/src/metabase/lib/query/test_spec.cljc b/src/metabase/lib/query/test_spec.cljcindex b4414f9266d..86d348212ce 100644--- a/src/metabase/lib/query/test_spec.cljc+++ b/src/metabase/lib/query/test_spec.cljc@@ -32,6 +32,7 @@[metabase.lib.temporal-bucket :as lib.temporal-bucket][metabase.util :as u][metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr][metabase.util.performance :refer [mapv]]))(mu/defn- find-source :- [:or ::lib.schema.metadata/table ::lib.schema.metadata/card]@@ -44,7 +45,7 @@(mu/defn- matches-column? :- :boolean[query :- ::lib.schema/query_stage-number :- :int- {:keys [name table-id source-name source-field-id display-name]} :- ::lib.schema.test-spec/test-column-spec+ {:keys [name table-id source-name source-field-id display-name]} :- ::lib.schema.test-spec/test-order-by-speccolumn :- ::lib.schema.metadata/column](cond-> (= name (:name column))(some? table-id) (and (= table-id (:table-id column)))@@ -56,7 +57,7 @@[query :- ::lib.schema/querystage-number :- :intavailable-columns :- [:sequential ::lib.schema.metadata/column]- column-spec :- ::lib.schema.test-spec/test-column-spec]+ column-spec :- ::lib.schema.test-spec/test-order-by-spec](let [columns (filterv (partial matches-column? query stage-number column-spec) available-columns)](case (count columns)0 (throw (ex-info "No column found" {:columns available-columns, :column-spec column-spec}))@@ -136,7 +137,7 @@(mu/defn- apply-binning :- [:or ::lib.schema.metadata/column ::lib.schema.ref/ref][query :- ::lib.schema/querystage-number :- :int- {:keys [unit bins bin-width]} :- ::lib.schema.test-spec/test-column-with-binning-spec+ {:keys [unit bins bin-width]} :- [:or ::lib.schema.test-spec/test-order-by-spec ::lib.schema.test-spec/test-join-source-spec]column :- [:or ::lib.schema.metadata/column ::lib.schema.ref/ref]](cond->> columnunit (add-temporal-bucket query stage-number unit)@@ -164,7 +165,7 @@(mu/defn- expression-spec->expression-parts[query :- ::lib.schema/querystage-number :- :int- expression-spec :- ::lib.schema.test-spec/test-expression-spec+ expression-spec :- [:or ::lib.schema.test-spec/test-expression-spec ::lib.schema.test-spec/test-join-source-spec]available-columns :- [:sequential ::lib.schema.metadata/column]](case (:type expression-spec):literal (:value expression-spec)@@ -176,7 +177,10 @@(:args expression-spec))}))(mu/defn- named-expression-spec? :- :boolean- [expression-spec :- [:or ::lib.schema.test-spec/test-expression-spec ::lib.schema.test-spec/test-named-expression-spec]]+ [expression-spec :- [:or+ ::lib.schema.test-spec/test-expression-spec+ ::lib.schema.test-spec/test-join-source-spec+ ::lib.schema.test-spec/test-named-expression-spec]](and(contains? expression-spec :name)(contains? expression-spec :value)@@ -185,7 +189,10 @@(mu/defn- expression-spec->expression-clause :- ::lib.schema.expression/expression[query :- ::lib.schema/querystage-number :- :int- {:keys [name value] :as expression-spec} :- [:or ::lib.schema.test-spec/test-expression-spec ::lib.schema.test-spec/test-named-expression-spec]+ {:keys [name value] :as expression-spec} :- [:or+ ::lib.schema.test-spec/test-expression-spec+ ::lib.schema.test-spec/test-join-source-spec+ ::lib.schema.test-spec/test-named-expression-spec]available-columns :- [:sequential ::lib.schema.metadata/column]](if (named-expression-spec? expression-spec)(-> (expression-spec->expression-clause query stage-number value available-columns)@@ -344,6 +351,10 @@fields (append-fields stage-number fields)limit (lib.limit/limit stage-number limit)))+(mr/def ::raw-spec+ "A test query spec as callers write it, possibly with string or camelCase keys and without defaults, before parsing."+ [:map {:closed false, ::mr/deliberately-open true, :description "an unparsed test query spec"}])+(def parse-query-spec"Parser for query-spec."(mc/decoder [:ref ::lib.schema.test-spec/test-query-spec]@@ -357,10 +368,10 @@(mu/defn test-query :- ::lib.schema/query"Creates a query from a test query spec."[metadata-providerable :- ::lib.schema.metadata/metadata-providerable- query-spec :- :any]+ query-spec :- ::raw-spec](let [{:keys [stages]} (parse-query-spec query-spec)- source (->> stages first :source (find-source metadata-providerable))- query (lib.query/query metadata-providerable source)]+ source (->> stages first :source (find-source metadata-providerable))+ query (lib.query/query metadata-providerable source)](reduce-kv append-stage-clauses query stages)))(mu/defn- field-id->field-ref :- :mbql.clause/field@@ -409,13 +420,14 @@(mtx/transformermtx/json-transformer(mtx/key-transformer {:decode #(-> % u/->kebab-case-en keyword)})- mtx/default-value-transformer- {:name :normalize})))+ {:name :normalize}+ mtx/strip-extra-keys-transformer+ mtx/default-value-transformer)))(mu/defn test-native-query :- ::lib.schema/query"Creates a native query from a test native query spec."[metadata-providerable :- ::lib.schema.metadata/metadata-providerable- native-query-spec :- :any]+ native-query-spec :- ::raw-spec](let [{:keys [query template-tags]} (parse-native-query-spec native-query-spec)](-> (lib.native/native-query metadata-providerable query)(add-template-tags template-tags))))diff --git a/src/metabase/lib/ref.cljc b/src/metabase/lib/ref.cljcindex 725669e078f..aba1f887e08 100644--- a/src/metabase/lib/ref.cljc+++ b/src/metabase/lib/ref.cljc@@ -2,9 +2,12 @@(:refer-clojure :exclude [ref])(:require[metabase.lib.dispatch :as lib.dispatch]+ [metabase.lib.schema.expression :as lib.schema.expression][metabase.lib.schema.id :as lib.schema.id]+ [metabase.lib.schema.metadata :as lib.schema.metadata][metabase.lib.schema.ref :as lib.schema.ref]- [metabase.util.malli :as mu]))+ [metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr]));;; TODO (Cam 6/12/25) -- update `ref-method` to take an options map instead of needing a dynamic variable for stuff;;; like this.@@ -20,10 +23,19 @@{:arglists '([x])}lib.dispatch/dispatch-value)+(mr/def ::referenceable+ "Something [[ref]] can create a fresh ref for: an MBQL clause, or column, metric, measure, or segment metadata."+ [:or+ ::lib.schema.expression/expression+ ::lib.schema.metadata/column+ ::lib.schema.metadata/metric+ ::lib.schema.metadata/measure+ ::lib.schema.metadata/segment])+(mu/defn ref :- ::lib.schema.ref/ref"Create a fresh ref that can be added to a query, e.g. a `:field`, `:aggregation`, or `:expression` reference. Willcreate a new UUID every time this is called."- [x :- some?]+ [x :- ::referenceable](ref-method x))(mu/defn field-ref-id :- [:maybe ::lib.schema.id/field]diff --git a/src/metabase/lib/referenced_columns.cljc b/src/metabase/lib/referenced_columns.cljcindex 5b7d15986a0..b1ff25471c8 100644--- a/src/metabase/lib/referenced_columns.cljc+++ b/src/metabase/lib/referenced_columns.cljc@@ -8,6 +8,7 @@[metabase.lib.dispatch :as lib.dispatch][metabase.lib.fe-util :as lib.fe-util][metabase.lib.schema :as lib.schema]+ [metabase.lib.schema.expression :as lib.schema.expression][metabase.lib.schema.metadata :as lib.schema.metadata][metabase.lib.util :as lib.util][metabase.util.malli :as mu]))@@ -35,10 +36,12 @@Throws if `x` is neither an MBQL clause nor a `:metadata/column` map. Schema validation isoff in production, so the runtime check is the gate."- ([query x] (referenced-columns query -1 x))+ ([query :- ::lib.schema/query+ x :- [:or ::lib.schema.metadata/column ::lib.schema.expression/expression]]+ (referenced-columns query -1 x))([query :- ::lib.schema/querystage-number :- :int- x]+ x :- [:or ::lib.schema.metadata/column ::lib.schema.expression/expression]](when-not (or (column-metadata? x) (lib.util/clause? x))(throw (ex-info "referenced-columns: expected an MBQL clause or :metadata/column"{:dispatch-value (lib.dispatch/dispatch-value x)diff --git a/src/metabase/lib/remove_replace.cljc b/src/metabase/lib/remove_replace.cljcindex 8fa16f6c16b..7afc198cf8c 100644--- a/src/metabase/lib/remove_replace.cljc+++ b/src/metabase/lib/remove_replace.cljc@@ -249,15 +249,15 @@query)))(mu/defn- remove-replace* :- :map- [query :- :map+ [query :- ::lib.schema/querystage-number :- :inttarget-clause :- ::lib.schema.mbql-clause/clauseremove-or-replace :- [:enum :remove :replace]replacement :- [:maybe [:or::lib.schema.mbql-clause/clause;; a metadata or `:lib/external-op` or something- [:map- [:lib/type qualified-keyword?]]]]]+ ::lib.metadata.calculation/displayable+ :metabase.lib.schema.common/external-op]]]{:pre [(vector? target-clause)]}(mu/disable-enforcement(let [target-clause (lib.common/->op-arg target-clause)@@ -311,8 +311,10 @@(mu/defn remove-clause :- ::lib.schema/query"Removes the `target-clause` from the stage specified by `stage-number` of `query`.If `stage-number` is not specified, the last stage is used."- ([query :- ::lib.schema/query- target-clause]+ ([query :- ::lib.schema/query+ target-clause :- [:or+ ::lib.schema.mbql-clause/clause+ ::lib.schema.join/join]](remove-clause query -1 target-clause))([query :- ::lib.schema/query@@ -527,7 +529,11 @@If you want to drop the old clause and replace it, that's [[remove-clause]] plus adding the new onewith [[lib.expression/expression]] and similar."- ([query target-clause new-clause]+ ([query :- ::lib.schema/query+ target-clause :- [:or+ ::lib.schema.mbql-clause/clause+ ::lib.schema.join/join]+ new-clause :- [:maybe ::lib.metadata.calculation/displayable]](replace-clause query -1 target-clause new-clause))([query :- ::lib.schema/query@@ -537,7 +543,7 @@::lib.schema.join/join];; replacement can be basically anything, an MBQL clause or literal allowed in MBQL, or `nil` (to remove it), or a;; join, or a metadata that can get turned into a clause.- new-clause :- :any]+ new-clause :- [:maybe ::lib.metadata.calculation/displayable]](cond(and (map? target-clause) (= (:lib/type target-clause) :mbql/join))(replace-join query stage-number target-clause new-clause)@@ -592,7 +598,9 @@If the specified join cannot be found, then `query` is returned as is.If renaming the join to `new-name` would clash with an existing join, asuffix is appended to `new-name` to make it unique."- ([query join-spec new-name]+ ([query :- ::lib.schema/query+ join-spec :- [:or ::lib.schema.join/join ::lib.schema.join/alias :int]+ new-name :- ::lib.schema.join/alias](rename-join query -1 join-spec new-name))([query :- ::lib.schema/query@@ -674,7 +682,8 @@If `stage-number` is not provided, the last stage is used.If the specified join cannot be found, then `query` is returned as is.Top level clauses containing references to the removed join are removed too."- ([query join-spec]+ ([query :- ::lib.schema/query+ join-spec :- [:or ::lib.schema.join/join :string :int]](remove-join query -1 join-spec))([query :- ::lib.schema/query@@ -700,13 +709,15 @@If `stage-number` is not provided, the last stage is used.If the specified join cannot be found, then `query` is returned as is.Top level clauses containing references to the removed join are removed too."- ([query join-spec new-join]+ ([query :- ::lib.schema/query+ join-spec :- [:or ::lib.schema.join/join :string :int]+ new-join :- [:maybe [:ref ::lib.join.util/join-with-optional-alias]]](replace-join query -1 join-spec new-join))([query :- ::lib.schema/querystage-number :- :intjoin-spec :- [:or ::lib.schema.join/join :string :int]- new-join]+ new-join :- [:maybe [:ref ::lib.join.util/join-with-optional-alias]]](if (nil? new-join)(remove-join query stage-number join-spec)(update-joins query stage-number join-spec@@ -781,7 +792,7 @@([query :- ::lib.schema/query](normalize-fields-clauses query nil))([query :- ::lib.schema/query- removed-location :- [:maybe [:sequential :any]]]+ removed-location :- [:maybe [:sequential [:or :keyword :int]]]](reduce #(normalize-fields-for-stage %1 %2 removed-location)query(range (count (:stages query))))))diff --git a/src/metabase/lib/schema.cljc b/src/metabase/lib/schema.cljcindex 0ae9d080fbe..94e06704bd7 100644--- a/src/metabase/lib/schema.cljc+++ b/src/metabase/lib/schema.cljc@@ -186,7 +186,7 @@source-field->names (update-vals source-field->refs(fn [field-refs](into #{}- (keep #(:source-field-name (lib.options/options %)))+ (keep (fn [[_tag opts]] (:source-field-name opts)))field-refs)))ignore-source-field-name? (fn [[_tag {:keys [source-field source-field-name], :as _opts} _id-or-name :as _ref]](when source-field-name@@ -576,22 +576,98 @@afterwards.In production these are legacy, `snake_cased` column metadata maps (i.e.- `:metabase.legacy-mbql.schema/legacy-column-metadata`), but this key is a verbatim snapshot that must survive- normalization untouched -- pointing at that schema here would rewrite its keys and drop deprecated ones, which- breaks round-tripping (see `metabase.lib.convert-test/round-trip-preserve-metadata-test`). So the elements are- declared as plain maps, which carry no decoders."- [:sequential :map])+ `:metabase.legacy-mbql.schema/legacy-column-metadata`), but some callers stash kebab-case, lib-flavored columns+ here instead (see `metabase.lib.convert-test/round-trip-preserve-metadata-test`); this key is a verbatim snapshot+ that must survive normalization untouched, so the elements mirror both key spellings without pointing at either+ source schema, which would rewrite keys and drop deprecated ones on decode."+ [:sequential+ [:map {:closed true}+ [:base_type {:optional true} [:maybe [:or :string :keyword]]]+ [:base-type {:optional true} [:maybe [:or :string :keyword]]]+ [:display_name {:optional true} [:maybe :string]]+ [:display-name {:optional true} [:maybe :string]]+ [:name {:optional true} [:maybe :string]]+ [:active {:optional true} [:maybe :boolean]]+ [:description {:optional true} [:maybe :string]]+ [:binning_info {:optional true} [:maybe+ [:map {:closed true}+ [:strategy {:optional true} [:maybe [:or :string :keyword]]]+ [:binning_strategy {:optional true} [:maybe [:or :string :keyword]]]+ [:bin_width {:optional true} [:maybe number?]]+ [:num_bins {:optional true} [:maybe :int]]+ [:min_value {:optional true} [:maybe number?]]+ [:max_value {:optional true} [:maybe number?]]]]]+ [:coercion_strategy {:optional true} [:maybe [:or :string :keyword]]]+ [:database_type {:optional true} [:maybe :string]]+ [:effective_type {:optional true} [:maybe [:or :string :keyword]]]+ [:effective-type {:optional true} [:maybe [:or :string :keyword]]]+ [:converted_timezone {:optional true} [:maybe :string]]+ [:field_ref {:optional true} [:maybe :metabase.legacy-mbql.schema/Reference]]+ [:field-ref {:optional true} [:maybe :metabase.legacy-mbql.schema/Reference]]+ [:fk_field_id {:optional true} [:maybe ::id/field]]+ [:fk-field-id {:optional true} [:maybe ::id/field]]+ [:fk_field_name {:optional true} [:maybe :string]]+ [:fk_join_alias {:optional true} [:maybe :string]]+ [:fk_target_field_id {:optional true} [:maybe ::id/field]]+ [:fingerprint {:optional true} [:maybe :metabase.lib.schema.metadata.fingerprint/fingerprint]]+ [:has_field_values {:optional true} [:maybe [:or :string :keyword]]]+ [:id {:optional true} [:maybe ::id/field]]+ [:inherited_temporal_unit {:optional true} [:maybe [:or :string :keyword]]]+ [:nfc_path {:optional true} [:maybe [:sequential :string]]]+ [:position {:optional true} [:maybe :int]]+ [:custom_position {:optional true} [:maybe :int]]+ [:database_position {:optional true} [:maybe :int]]+ [:data_sensitivity {:optional true} [:maybe [:or :string :keyword]]]+ [:fingerprint_version {:optional true} [:maybe :int]]+ [:parent_id {:optional true} [:maybe ::id/field]]+ [:points_of_interest {:optional true} [:maybe :string]]+ [:preview_display {:optional true} [:maybe :boolean]]+ [:caveats {:optional true} [:maybe :string]]+ [:target {:optional true} [:maybe [:or ::id/field :metabase.legacy-mbql.schema/Reference]]]+ [:options {:optional true} [:maybe :metabase.lib.schema.metadata/column.options]]+ [:remapped_from {:optional true} [:maybe :string]]+ [:remapped_to {:optional true} [:maybe :string]]+ [:remapped_from_index {:optional true} [:maybe :int]]+ [:source_alias {:optional true} [:maybe :string]]+ [:aggregation_index {:optional true} [:maybe :int]]+ [:expression_name {:optional true} [:maybe :string]]+ [:special_type {:optional true} [:maybe [:or :string :keyword]]]+ [:remapping {:optional true} [:maybe [:map-of [:ref :metabase.lib.schema.literal/literal]+ [:ref :metabase.lib.schema.literal/literal]]]]+ [:selected? {:optional true} [:maybe :boolean]]+ [:semantic_type {:optional true} [:maybe [:or :string :keyword]]]+ [:semantic-type {:optional true} [:maybe [:or :string :keyword]]]+ [:settings {:optional true} [:maybe :metabase.lib.schema.common/visualization-settings]]+ [:source {:optional true} [:maybe [:or :string :keyword]]]+ [:table_id {:optional true} [:maybe ::id/table]]+ [:table-id {:optional true} [:maybe ::id/table]]+ [:unit {:optional true} [:maybe [:or :string :keyword]]]+ [:visibility_type {:optional true} [:maybe [:or :string :keyword]]]+ [:visibility-type {:optional true} [:maybe [:or :string :keyword]]]+ [:lib/join-alias {:optional true} [:maybe :string]]+ [:lib/source {:optional true} [:maybe [:ref ::lib.schema.metadata/column.source]]]+ [:lib/breakout? {:optional true} [:maybe :boolean]]+ [:lib/source-column-alias {:optional true} [:maybe :string]]+ [:lib/desired-column-alias {:optional true} [:maybe :string]]+ [:lib/original-name {:optional true} [:maybe :string]]+ [:lib/deduplicated-name {:optional true} [:maybe :string]]+ [:lib/original-display-name {:optional true} [:maybe :string]]+ [:lib/simple-display-name {:optional true} [:maybe :string]]+ [:lib/transformation-added-base-type {:optional true} [:maybe :boolean]]+ [:database_partitioned {:optional true} [:maybe :boolean]]+ [:lib/expression-name {:optional true} [:maybe :string]]+ [:lib/external-remap {:optional true} [:maybe [:ref ::lib.schema.metadata/column.remapping.external]]]+ [:lib/original-fk-field-id {:optional true} [:maybe ::id/field]]+ [:lib/original-join-alias {:optional true} [:maybe :string]]+ [:lib/card-id {:optional true} [:maybe ::id/card]]+ [:lib/from-model? {:optional true} [:maybe :boolean]]+ [:lib/temporal-unit {:optional true} [:maybe [:or :string :keyword]]]]])(mr/def ::query.snapshot- "A whole copy of a query stashed on the query itself under one of the internal keys below.-- Deliberately NOT `[:ref ::query]`: [[malli.util/merge]] recurses into the entries that the two schemas being merged- share, so a self-referential `::query` entry makes `[:merge ::query [:map [<same key> ::query]]]` -- which- [[metabase.query-processor.util.add-alias-info]] uses -- recur until the stack blows. This shallow shape says what- the value is without reintroducing the cycle."- [:map- [:lib/type [:= {:decode/normalize common/normalize-keyword} :mbql/query]]- [:stages [:ref ::stages]]])+ "A whole copy of a query stashed on the query itself under one of the internal keys below: the same shape as+ [[::query]] minus the `:and`-level constraints, so validating the stashed copy does not re-run uniqueness checks+ against the query it was copied from."+ [:ref ::query.map])(mr/def ::cache-strategy.nocache[:map {:closed true, :decode/normalize common/normalize-map-no-kebab-case}@@ -638,6 +714,30 @@[:duration [:merge ::cache-strategy.duration ::cache-strategy.on-query]][:schedule [:merge ::cache-strategy.schedule ::cache-strategy.on-query]]])+(mr/def ::native-query-document-value+ "A value inside a driver's compiled native query document: a scalar, or a nested document/array of them. Covers+ every native query shape drivers in this codebase actually produce -- SQL text (a bare `:string`) as well as+ document-pipeline languages like Mongo's (a `:map-of`/`:sequential` tree)."+ [:schema+ {:registry+ {::value [:or+ :nil+ :boolean+ :string+ :keyword+ number?+ [:sequential [:ref ::value]]+ [:map-of :string [:ref ::value]]]}}+ ::value])++(mr/def ::driver-native-query+ "A driver's native query: a native query document, or a form in the driver's own shape such as a keyword-keyed map decoded from a JSON request."+ [:or+ ::native-query-document-value+ [:schema {::mr/deliberately-open true+ :description "a driver's native query in the driver's own shape, e.g. a keyword-keyed map from a JSON request"}+ :some]])+(mr/def ::compiled-native-query"A native query compiled from this query by [[metabase.query-processor.compile]], ready to hand to the driver.`:query` is whatever native form the driver uses -- a string for SQL drivers, a map for Mongo -- so it is only@@ -646,93 +746,100 @@Deliberately NOT closed: drivers add their own keys, e.g. Mongo adds `:collection`, `:projections` and `:mbql?`(see `:metabase.driver.mongo.query-processor/compiled-pipeline`), and for a query that was already native thecompiled form is the native stage itself, carrying every key a `::stage.native` has."- [:map- [:query :some]+ [:map {::mr/deliberately-open true+ :description "drivers add their own keys to a compiled query, and an already-native query compiles to its whole native stage"}+ [:query ::driver-native-query][:params {:optional true} [:maybe [:sequential [:ref ::literal/param-value]]]]])-(mr/def ::query- [:and- [:map- {:description "Valid MBQL 5 query."- :decode/normalize #'normalize-query- :decode/api #'common/remove-internal-keys- :encode/serialize #'serialize-query- :encode/for-hashing #'encode-query-for-hashing- :closed true}- [:lib/type [:=- {:decode/normalize common/normalize-keyword, :default :mbql/query}- :mbql/query]]- ;; TODO (Cam 6/12/25) -- why in the HECC is `:lib/metadata` not a required key here? It's virtually REQUIRED for- ;; anything to work correctly outside of the low-level conversion code. We should make it required and then fix- ;; whatever breaks.- [:lib/metadata {:optional true} ::lib.schema.metadata/metadata-provider]- [:database {:optional true} [:multi {:dispatch (partial = id/saved-questions-virtual-database-id)}- [true ::id/saved-questions-virtual-database]- [false ::id/database]]]- [:stages [:ref ::stages]]- [:parameters {:optional true} [:ref ::lib.schema.parameter/parameters]]- ;;- ;; OPTIONS- ;;- ;; These keys are used to tweak behavior of the Query Processor.- ;;- [:settings {:optional true} [:ref ::lib.schema.settings/settings]]- [:constraints {:optional true} [:ref ::lib.schema.constraints/constraints]]- [:middleware {:optional true} [:ref ::lib.schema.middleware-options/middleware-options]]- [:async? {:optional true} :boolean]- [:cache-strategy {:optional true} [:maybe [:ref ::cache-strategy]]]- [:was-pivot- {:optional true- :description- "Whether this query was originally run as a pivot query. Stamped into the saved `json_query` by+(def ^:private query-map+ [:map+ {:description "Valid MBQL 5 query."+ :decode/normalize #'normalize-query+ :decode/api #'common/remove-internal-keys+ :encode/serialize #'serialize-query+ :encode/for-hashing #'encode-query-for-hashing+ :closed true}+ [:lib/type [:=+ {:decode/normalize common/normalize-keyword, :default :mbql/query}+ :mbql/query]]+ ;; TODO (Cam 6/12/25) -- why in the HECC is `:lib/metadata` not a required key here? It's virtually REQUIRED for+ ;; anything to work correctly outside of the low-level conversion code. We should make it required and then fix+ ;; whatever breaks.+ [:lib/metadata {:optional true} ::lib.schema.metadata/metadata-provider]+ [:database {:optional true} [:multi {:dispatch (partial = id/saved-questions-virtual-database-id)}+ [true ::id/saved-questions-virtual-database]+ [false ::id/database]]]+ [:stages [:ref ::stages]]+ [:parameters {:optional true} [:ref ::lib.schema.parameter/parameters]]+ ;;+ ;; OPTIONS+ ;;+ ;; These keys are used to tweak behavior of the Query Processor.+ ;;+ [:settings {:optional true} [:ref ::lib.schema.settings/settings]]+ [:constraints {:optional true} [:ref ::lib.schema.constraints/constraints]]+ [:middleware {:optional true} [:ref ::lib.schema.middleware-options/middleware-options]]+ [:async? {:optional true} :boolean]+ [:cache-strategy {:optional true} [:maybe [:ref ::cache-strategy]]]+ [:was-pivot+ {:optional true+ :description+ "Whether this query was originally run as a pivot query. Stamped into the saved `json_query` by`metabase.query-processor.middleware.process-userland-query` and sent back by clients re-downloading pivot queryresults."}- [:maybe :boolean]]- [:pivot-rows {:optional true} [:maybe [:sequential [:int {:min 0}]]]]- [:pivot-cols {:optional true} [:maybe [:sequential [:int {:min 0}]]]]- [:pivot-measures {:optional true} [:maybe [:sequential [:int {:min 0}]]]]- [:show-row-totals {:optional true} [:maybe :boolean]]- [:show-column-totals {:optional true} [:maybe :boolean]]- [:pivot_rows {:optional true} [:maybe [:sequential [:int {:min 0}]]]]- [:pivot_cols {:optional true} [:maybe [:sequential [:int {:min 0}]]]]- [:pivot_measures {:optional true} [:maybe [:sequential [:int {:min 0}]]]]- [:show_row_totals {:optional true} [:maybe :boolean]]- [:show_column_totals {:optional true} [:maybe :boolean]]- [:viz-settings {:optional true} [:maybe [:ref ::common/visualization-settings]]]- [:user-parameters {:optional true} [:ref ::lib.schema.parameter/parameters]]- [:lib.convert/converted? {:optional true} :boolean]- [:qp/compiled {:optional true} [:ref ::compiled-native-query]]- [:qp/compiled-inline {:optional true} [:ref ::compiled-native-query]]- [:qp/source-card-id {:optional true} [:ref ::id/card]]- [:qp/skip-result-metadata-persistence {:optional true} :boolean]- [:qp.pivot/unremapped-breakout-combination {:optional true} [:sequential [:int {:min 0}]]]- [:qp.pivot/remapped-breakout-combination {:optional true} [:maybe [:sequential [:int {:min 0}]]]]- [:qp.pivot/num-remapped-cols {:optional true} [:int {:min 0}]]- [:qp.pivot/num-unremapped-breakouts {:optional true} [:int {:min 0}]]- [:qp.pivot/num-remapped-breakouts {:optional true} [:int {:min 0}]]- [:qp.pivot/remapped-indexes {:optional true} [:map-of [:int {:min 0}] [:int {:min 0}]]]- [:query-permissions/referenced-card-ids {:optional true} [:maybe [:set [:ref ::id/card]]]]- [:destination-database/id {:optional true} [:ref ::id/database]]- [:impersonation/role {:optional true} ::common/non-blank-string]- [:impersonation/admin? {:optional true} :boolean]- [:impersonation/allow-write? {:optional true} :boolean]- [:metabase.query-processor.util.add-alias-info/original {:optional true} [:ref ::query.snapshot]]- [:metabase.query-processor.middleware.add-remaps/external-remaps {:optional true} [:ref ::external-remappings]]- [:metabase-enterprise.sandbox.query-processor.middleware.sandboxing/original-metadata- {:optional true}- [:ref ::sandboxing.original-metadata]]- ;;- ;; INFO- ;;- ;; Used when recording info about this run in the QueryExecution log; things like context query was ran in and- ;; User who ran it- [:info {:optional true} [:ref ::info/info]]- ;;- ;; ACTIONS- ;;- ;; This stuff is only used for Actions.- [:create-row {:optional true} [:ref ::actions/row]]- [:update-row {:optional true} [:ref ::actions/row]]]+ [:maybe :boolean]]+ [:pivot-rows {:optional true} [:maybe [:sequential [:int {:min 0}]]]]+ [:pivot-cols {:optional true} [:maybe [:sequential [:int {:min 0}]]]]+ [:pivot-measures {:optional true} [:maybe [:sequential [:int {:min 0}]]]]+ [:show-row-totals {:optional true} [:maybe :boolean]]+ [:show-column-totals {:optional true} [:maybe :boolean]]+ [:pivot_rows {:optional true} [:maybe [:sequential [:int {:min 0}]]]]+ [:pivot_cols {:optional true} [:maybe [:sequential [:int {:min 0}]]]]+ [:pivot_measures {:optional true} [:maybe [:sequential [:int {:min 0}]]]]+ [:show_row_totals {:optional true} [:maybe :boolean]]+ [:show_column_totals {:optional true} [:maybe :boolean]]+ [:viz-settings {:optional true} [:maybe [:ref ::common/visualization-settings]]]+ [:user-parameters {:optional true} [:ref ::lib.schema.parameter/parameters]]+ [:lib.convert/converted? {:optional true} :boolean]+ [:qp/compiled {:optional true} [:ref ::compiled-native-query]]+ [:qp/compiled-inline {:optional true} [:ref ::compiled-native-query]]+ [:qp/source-card-id {:optional true} [:ref ::id/card]]+ [:qp/skip-result-metadata-persistence {:optional true} :boolean]+ [:qp.pivot/unremapped-breakout-combination {:optional true} [:sequential [:int {:min 0}]]]+ [:qp.pivot/remapped-breakout-combination {:optional true} [:maybe [:sequential [:int {:min 0}]]]]+ [:qp.pivot/num-remapped-cols {:optional true} [:int {:min 0}]]+ [:qp.pivot/num-unremapped-breakouts {:optional true} [:int {:min 0}]]+ [:qp.pivot/num-remapped-breakouts {:optional true} [:int {:min 0}]]+ [:qp.pivot/remapped-indexes {:optional true} [:map-of [:int {:min 0}] [:int {:min 0}]]]+ [:query-permissions/referenced-card-ids {:optional true} [:maybe [:set [:ref ::id/card]]]]+ [:destination-database/id {:optional true} [:ref ::id/database]]+ [:impersonation/role {:optional true} ::common/non-blank-string]+ [:impersonation/admin? {:optional true} :boolean]+ [:impersonation/allow-write? {:optional true} :boolean]+ [:metabase.query-processor.util.add-alias-info/original {:optional true} [:ref ::query.snapshot]]+ [:metabase.query-processor.middleware.add-remaps/external-remaps {:optional true} [:ref ::external-remappings]]+ [:metabase-enterprise.sandbox.query-processor.middleware.sandboxing/original-metadata+ {:optional true}+ [:ref ::sandboxing.original-metadata]]+ ;;+ ;; INFO+ ;;+ ;; Used when recording info about this run in the QueryExecution log; things like context query was ran in and+ ;; User who ran it+ [:info {:optional true} [:ref ::info/info]]+ ;;+ ;; ACTIONS+ ;;+ ;; This stuff is only used for Actions.+ [:create-row {:optional true} [:ref ::actions/row]]+ [:update-row {:optional true} [:ref ::actions/row]]])++(mr/def ::query.map+ query-map)++(mr/def ::query+ [:and+ query-map;;;; CONSTRAINTS[:ref ::lib.schema.util/unique-uuids]diff --git a/src/metabase/lib/schema/aggregation.cljc b/src/metabase/lib/schema/aggregation.cljcindex 07cef49e25d..ace2a258ca4 100644--- a/src/metabase/lib/schema/aggregation.cljc+++ b/src/metabase/lib/schema/aggregation.cljc@@ -287,7 +287,7 @@:description (i18n/tru "Maximum value of a column")})}])(mr/def ::operator- [:map+ [:map {:closed true}[:lib/type [:= :operator/aggregation]][:short (into [:enum] (map :short) aggregation-operators)][:supported-field {:optional true} [:maybe :keyword]] ; TODO more precise type?diff --git a/src/metabase/lib/schema/binning.cljc b/src/metabase/lib/schema/binning.cljcindex 98d1e2f29d7..e7af2cc2bec 100644--- a/src/metabase/lib/schema/binning.cljc+++ b/src/metabase/lib/schema/binning.cljc@@ -74,8 +74,9 @@[:metadata-fn {:optional true} ifn?]]])(mr/def ::binning-option- [:map+ [:map {:closed true}[:lib/type [:= :option/binning]][:display-name :string][:mbql [:maybe ::binning]]- [:default {:optional true} :boolean]])+ [:default {:optional true} :boolean]+ [:selected {:optional true} :boolean]])diff --git a/src/metabase/lib/schema/common.cljc b/src/metabase/lib/schema/common.cljcindex a9831613619..53e5f90d6ba 100644--- a/src/metabase/lib/schema/common.cljc+++ b/src/metabase/lib/schema/common.cljc@@ -96,17 +96,44 @@(cond-> x(keyword? x) u/qualified-name))+(mr/def ::clause-arg+ "One arg of an MBQL clause in any normalization state: a literal, a keyword such as a temporal unit, an options map+ (possibly string-keyed), or a nested clause or sequence of clauses."+ [:or+ :nil+ :keyword+ :metabase.lib.schema.literal/literal+ [:ref ::clause-options]+ [:sequential [:ref ::clause-arg]]])++(mr/def ::any-clause+ "An MBQL clause `[tag & args]` in any normalization state -- `tag` a keyword, or a string before normalization+ keywordizes it -- e.g. not yet uuid'd, or still carrying legacy args."+ [:cat [:or :keyword :string] [:* ::clause-arg]])++(mr/def ::possibly-unnormalized-clause+ "A (possibly not-yet-normalized) MBQL clause `[tag & args]` -- `tag` a keyword, or a string before normalization+ keywordizes it -- or a literal value."+ [:or+ :metabase.lib.schema.literal/literal+ ::any-clause])++(mr/def ::clause-tag-candidate+ "Any value a `:multi` schema dispatches on with [[mbql-clause-tag]], which may or may not be an MBQL clause."+ [:schema {::mr/deliberately-open true, :description "a value that may be an MBQL clause"} :any])+(mu/defn mbql-clause-tag :- [:maybe :keyword]"If `x` is a (possibly not-yet-normalized) MBQL clause, return its `tag`."- [x]+ [x :- ::clause-tag-candidate](when (and (vector? x)((some-fn keyword? string?) (first x)))(keyword (first x))));;; TODO (Cam 9/8/25) -- overlapping functionality with [[metabase.lib.util/clause-of-type?]]-(mu/defn is-clause?+(mu/defn is-clause? :- :boolean"Whether `x` is a (possibly not-yet-normalized) MBQL clause with `tag`. Does not check that the clause is valid."- [tag :- :keyword x]+ [tag :- :keyword+ x :- ::clause-tag-candidate](= (mbql-clause-tag x) tag))(mr/def ::non-blank-string@@ -133,6 +160,27 @@only."[:map {:closed false, ::mr/deliberately-open true, :description "visualization settings", :decode/normalize normalize-map-no-kebab-case}])+(mr/def ::clause-options+ "The options map of any MBQL clause, MBQL 5 or legacy, possibly not yet normalized; its keys depend on the clause."+ [:map {:closed false, ::mr/deliberately-open true, :description "options map of any MBQL clause"}])++(mr/def ::database-details+ "Connection details for a Database; the `.cljc` equivalent of [[metabase.util.malli.schema/DatabaseDetails]]."+ [:map {:closed false, ::mr/deliberately-open true, :description "database connection details"}])++(mr/def ::database-settings+ "A Database's `:settings`; the `.cljc` equivalent of [[metabase.util.malli.schema/DatabaseSettings]]."+ [:map {:closed false, ::mr/deliberately-open true, :description "database settings"}])++(mr/def ::exception-data+ "The `ex-data` of an exception; the `.cljc` equivalent of [[metabase.util.malli.schema/ExceptionData]]."+ [:map {:closed false, ::mr/deliberately-open true, :description "exception data"}])++(mr/def ::field-value+ "One value of a Field; the `.cljc` equivalent of [[metabase.util.malli.schema/FieldValue]]."+ [:maybe [:or :string number? :boolean uuid? #?(:clj [:fn {:error/message "instance of java.time.temporal.Temporal"}+ #(instance? java.time.temporal.Temporal %)])]])+(mr/def ::uuid[:string{:decode/normalize (fn [x]@@ -186,7 +234,7 @@;;; will throw in dev. See [[metabase.lib.schema.common-test/normalize-base-type-test]] for more info(mu/defn- normalize-base-type* :- [:maybe [:ref ::base-type]]- [x]+ [x :- [:or :nil :string :keyword]](normalize-keyword x))(defn normalize-base-type@@ -289,7 +337,19 @@{:error/message \":native is not allowed in an MBQL stage\":decode/normalize #(cond-> % (map? %) (dissoc :native))}#(not (when (map? %) (contains? :native)))]]"- [k->message :- [:map-of :keyword :string]]+ [k->message :- [:map-of [:enum+ :query :source-table :source-card :fields :filter :filters :breakout :aggregation+ :limit :order-by :offset :page :args :native :aggregation-idents :breakout-idents+ :expression-idents :source-metadata :source-query :type :database :lib/options+ :expressions :pivot :joins :ident :lib/expression-name :lib/join-alias :fk-field-id+ :binning :field-ref :model/inner-ident :source :source-alias :unit+ :metabase.lib.join/join-alias :metabase.lib.field/binning+ :metabase.lib.field/temporal-unit :metabase.lib.field/original-effective-type+ :metabase.lib.field/simple-display-name+ :metabase.lib.query/transformation-added-base-type+ :lib/stage-metadata :condition :parameters :dimension :strategy :lib/uuid :lib/type+ :model/inner_ident :stages :lib/model-display-name]+ :string]](let [fn-schemas (map (fn [[k message]][:fn{:error/message message@@ -369,18 +429,23 @@[:metabase.driver.sql.parameters.substitution/compiling-field-filter? {:optional true} :boolean][:metabase.driver.sqlserver/optimized-bucketing? {:optional true} :boolean][:metabase.driver.mongo.query-processor/join-local {:optional true} [:ref :metabase.lib.schema.join/alias]]- [:metabase.mcp.v2.query/keyset {:optional true} :boolean]]+ [:metabase.mcp.v2.query/keyset {:optional true} :boolean]+ [:temporal-unit {:optional true} [:ref :metabase.lib.schema.temporal-bucketing/unit]]+ [:inherited-temporal-unit {:optional true} [:ref :metabase.lib.schema.temporal-bucketing/unit]]+ [:lib/original-effective-type {:optional true} [:maybe ::base-type]]+ [:lib/transformation-added-base-type {:optional true} [:maybe :boolean]]+ [:source-field {:optional true} [:ref :metabase.lib.schema.id/field]]](disallowed-keys{:ident ":ident is deprecated and should not be included in options maps"})])(mr/def ::external-op- [:map+ [:map {:closed true}[:lib/type [:= :lib/external-op]][:operator [:multi {:dispatch string?}[true :string][false :keyword]]]- [:args [:schema {:decode/normalize vec} [:sequential :any]]]- [:options {:optional true} ::options]])+ [:args [:schema {:decode/normalize vec} [:sequential [:ref :metabase.lib.common/op-arg]]]]+ [:options {:optional true} [:maybe [:ref ::clause-options]]]])#?(:clj(defn- instance-of-class* [& classes]diff --git a/src/metabase/lib/schema/drill_thru.cljc b/src/metabase/lib/schema/drill_thru.cljcindex 769102e8860..9da0c529767 100644--- a/src/metabase/lib/schema/drill_thru.cljc+++ b/src/metabase/lib/schema/drill_thru.cljc@@ -11,6 +11,7 @@[metabase.lib.schema.expression :as lib.schema.expression][metabase.lib.schema.extraction :as lib.schema.extraction][metabase.lib.schema.id :as lib.schema.id]+ [metabase.lib.schema.literal :as lib.schema.literal][metabase.lib.schema.metadata :as lib.schema.metadata][metabase.lib.schema.order-by :as lib.schema.order-by][metabase.lib.schema.ref :as lib.schema.ref]@@ -30,7 +31,7 @@(= (namespace k) "drill-thru")))])(mr/def ::drill-thru.common- [:map+ [:map {:closed true}[:type ::drill-thru.type][:lib/type [:= :metabase.lib.drill-thru/drill-thru]]])@@ -49,7 +50,7 @@;;; https://metaboat.slack.com/archives/C04CYTEL9N2/p1701803047600169 for more information. -- Cam(mr/def ::drill-thru.object-details.dimension- [:map+ [:map {:closed true}[:column [:ref ::lib.schema.metadata/column]];; we should ignore NULL values for PKs and FKs -- do not add filters on them.[:value [:and@@ -74,11 +75,11 @@(mr/def ::drill-thru.fk-details[:merge- ::drill-thru.common.with-column+ ::drill-thru.common[:map[:type [:= :drill-thru/fk-details]][:column [:ref ::drill-thru.fk-details.fk-column]]- [:object-id :any]+ [:object-id [:ref ::lib.schema.literal/param-value]][:many-pks? :boolean]]])(mr/def ::drill-thru.zoom@@ -86,13 +87,13 @@::drill-thru.common.with-column[:map[:type [:= :drill-thru/zoom]]- [:object-id :any]+ [:object-id [:ref ::lib.schema.literal/param-value]];; TODO -- I don't think we really need this because there is no situation in which this isn't `false`, if it were;; true we'd return a `::drill-thru.pk` drill instead. See if we can remove this key without breaking the FE.[:many-pks? [:= false]]]])(mr/def ::drill-thru.quick-filter.operator- [:map+ [:map {:closed true}[:name ::lib.schema.common/non-blank-string][:filter [:ref ::lib.schema.expression/boolean]]])@@ -103,7 +104,7 @@[:type [:= :drill-thru/quick-filter]][:operators [:sequential ::drill-thru.quick-filter.operator]][:column [:ref ::lib.schema.metadata/column]]- [:value [:maybe :any]]+ [:value [:maybe [:or [:= :null] ::lib.schema.literal/param-value]]][:query [:ref ::lib.schema/query]][:stage-number number?]]])@@ -127,12 +128,13 @@::drill-thru.common[:map[:type [:= :drill-thru/pivot]]+ [:dimensions {:optional true} [:maybe [:ref ::context.row]]][:pivots [:map-of ::pivot-types [:sequential [:ref ::lib.schema.metadata/column]]]][:stage-number number?]]])(mr/def ::drill-thru.sort[:merge- ::drill-thru.common+ ::drill-thru.common.with-column[:map[:type [:= :drill-thru/sort]][:sort-directions [:sequential ::lib.schema.order-by/direction]]]])@@ -160,7 +162,6 @@::drill-thru.common.with-column[:map[:type [:= :drill-thru/column-filter]]- [:column [:ref ::lib.schema.metadata/column]][:query [:ref ::lib.schema/query]][:stage-number number?]]])@@ -169,6 +170,7 @@::drill-thru.common.with-column[:map[:type [:= :drill-thru/column-extract]]+ [:display-name :string][:query [:ref ::lib.schema/query]][:stage-number number?][:extractions [:sequential [:ref ::lib.schema.extraction/extraction]]]]])@@ -177,22 +179,24 @@[:merge::drill-thru.common.with-column[:map- [:type [:= :drill-thru/combine-columns]]]])+ [:type [:= :drill-thru/combine-columns]]+ [:query [:ref ::lib.schema/query]]+ [:stage-number number?]]])-;;; TODO FIXME -- it seems like underlying records drills also include `:dimensions` and `:column-ref`...-;;; see [[metabase.lib.drill-thru.underlying-records/underlying-records-drill]]... this should be part of the schema(mr/def ::drill-thru.underlying-records[:merge::drill-thru.common[:map[:type [:= :drill-thru/underlying-records]][:row-count number?]- [:table-name [:maybe string?]]]])+ [:table-name [:maybe string?]]+ [:dimensions {:optional true} [:maybe [:ref ::context.row]]]+ [:column-ref {:optional true} [:maybe [:ref ::lib.schema.ref/ref]]]]])(mr/def ::drill-thru.automatic-insights[:merge::drill-thru.common- [:map+ [:map {:closed true}[:type [:= :drill-thru/automatic-insights]][:lib/type [:= :metabase.lib.drill-thru/drill-thru]][:column-ref [:maybe [:ref ::lib.schema.ref/ref]]]@@ -206,6 +210,7 @@::drill-thru.common[:map[:type [:= :drill-thru/zoom-in.timeseries]]+ [:display-name :string][:dimension [:ref ::context.row.value]][:next-unit [:ref ::drill-thru.zoom-in.timeseries.next-unit]]]])@@ -241,12 +246,13 @@[:map[:type [:= :drill-thru/zoom-in.geographic]][:subtype [:= :drill-thru.zoom-in.geographic/country-state-city->binned-lat-lon]]+ [:display-name :string][:column ::drill-thru.zoom-in.geographic.column.county-state-city]- [:value some?]- [:latitude [:map+ [:value ::lib.schema.literal/param-value]+ [:latitude [:map {:closed true}[:column [:ref ::drill-thru.zoom-in.geographic.column.latitude]][:bin-width [:ref ::lib.schema.binning/bin-width]]]]- [:longitude [:map+ [:longitude [:map {:closed true}[:column [:ref ::drill-thru.zoom-in.geographic.column.longitude]][:bin-width [:ref ::lib.schema.binning/bin-width]]]]]])@@ -256,31 +262,26 @@[:map[:type [:= :drill-thru/zoom-in.geographic]][:subtype [:= :drill-thru.zoom-in.geographic/binned-lat-lon->binned-lat-lon]]- [:latitude [:map+ [:display-name :string]+ [:latitude [:map {:closed true}[:column [:ref ::drill-thru.zoom-in.geographic.column.latitude]][:bin-width [:ref ::lib.schema.binning/bin-width]][:min number?][:max number?]]]- [:longitude [:map+ [:longitude [:map {:closed true}[:column [:ref ::drill-thru.zoom-in.geographic.column.longitude]][:bin-width [:ref ::lib.schema.binning/bin-width]][:min number?][:max number?]]]]])(mr/def ::drill-thru.zoom-in.geographic- [:and- [:merge- ::drill-thru.common- [:map- [:type [:= :drill-thru/zoom-in.geographic]]- [:subtype keyword?]]]- [:multi {:dispatch :subtype- :error/fn (fn [{:keys [value]} _]- (str "Invalid zoom-in.geographic drill thru subtype" (pr-str value)))}- [:drill-thru.zoom-in.geographic/country-state-city->binned-lat-lon- ::drill-thru.zoom-in.geographic.country-state-city->binned-lat-lon]- [:drill-thru.zoom-in.geographic/binned-lat-lon->binned-lat-lon- ::drill-thru.zoom-in.geographic.binned-lat-lon->binned-lat-lon]]])+ [:multi {:dispatch :subtype+ :error/fn (fn [{:keys [value]} _]+ (str "Invalid zoom-in.geographic drill thru subtype" (pr-str value)))}+ [:drill-thru.zoom-in.geographic/country-state-city->binned-lat-lon+ ::drill-thru.zoom-in.geographic.country-state-city->binned-lat-lon]+ [:drill-thru.zoom-in.geographic/binned-lat-lon->binned-lat-lon+ ::drill-thru.zoom-in.geographic.binned-lat-lon->binned-lat-lon]])(mr/def ::drill-thru.zoom-in.binning[:merge@@ -292,29 +293,27 @@[:new-binning ::lib.schema.binning/binning]]])(mr/def ::drill-thru- [:and- ::drill-thru.common- [:multi {:dispatch :type- :error/fn (fn [{:keys [value]} _]- (str "Invalid drill thru (unknown :type): " (pr-str value)))}- [:drill-thru/pk ::drill-thru.pk]- [:drill-thru/fk-details ::drill-thru.fk-details]- [:drill-thru/zoom ::drill-thru.zoom]- [:drill-thru/quick-filter ::drill-thru.quick-filter]- [:drill-thru/fk-filter ::drill-thru.fk-filter]- [:drill-thru/distribution ::drill-thru.distribution]- [:drill-thru/pivot ::drill-thru.pivot]- [:drill-thru/sort ::drill-thru.sort]- [:drill-thru/summarize-column ::drill-thru.summarize-column]- [:drill-thru/summarize-column-by-time ::drill-thru.summarize-column-by-time]- [:drill-thru/column-filter ::drill-thru.column-filter]- [:drill-thru/column-extract ::drill-thru.column-extract]- [:drill-thru/combine-columns ::drill-thru.combine-columns]- [:drill-thru/underlying-records ::drill-thru.underlying-records]- [:drill-thru/automatic-insights ::drill-thru.automatic-insights]- [:drill-thru/zoom-in.timeseries ::drill-thru.zoom-in.timeseries]- [:drill-thru/zoom-in.geographic ::drill-thru.zoom-in.geographic]- [:drill-thru/zoom-in.binning ::drill-thru.zoom-in.binning]]])+ [:multi {:dispatch :type+ :error/fn (fn [{:keys [value]} _]+ (str "Invalid drill thru (unknown :type): " (pr-str value)))}+ [:drill-thru/pk ::drill-thru.pk]+ [:drill-thru/fk-details ::drill-thru.fk-details]+ [:drill-thru/zoom ::drill-thru.zoom]+ [:drill-thru/quick-filter ::drill-thru.quick-filter]+ [:drill-thru/fk-filter ::drill-thru.fk-filter]+ [:drill-thru/distribution ::drill-thru.distribution]+ [:drill-thru/pivot ::drill-thru.pivot]+ [:drill-thru/sort ::drill-thru.sort]+ [:drill-thru/summarize-column ::drill-thru.summarize-column]+ [:drill-thru/summarize-column-by-time ::drill-thru.summarize-column-by-time]+ [:drill-thru/column-filter ::drill-thru.column-filter]+ [:drill-thru/column-extract ::drill-thru.column-extract]+ [:drill-thru/combine-columns ::drill-thru.combine-columns]+ [:drill-thru/underlying-records ::drill-thru.underlying-records]+ [:drill-thru/automatic-insights ::drill-thru.automatic-insights]+ [:drill-thru/zoom-in.timeseries ::drill-thru.zoom-in.timeseries]+ [:drill-thru/zoom-in.geographic ::drill-thru.zoom-in.geographic]+ [:drill-thru/zoom-in.binning ::drill-thru.zoom-in.binning]]);;;;;; ## Context@@ -346,7 +345,7 @@;;; | Pivot "Agg" Cell | | | ✔ | ✔ |(mr/def ::context.row.value- [:map+ [:map {:closed true}[:column [:ref ::lib.schema.metadata/column]][:column-ref [:ref ::lib.schema.ref/ref]][:value [:fn@@ -360,10 +359,13 @@[:sequential [:ref ::context.row.value]])(mr/def ::context- [:map+ [:map {:closed true}[:column [:maybe [:ref ::lib.schema.metadata/column]]][:column-ref [:maybe [:ref ::lib.schema.ref/ref]]]- [:value [:maybe :any]]+ [:value [:maybe [:or+ [:= :null]+ ::lib.schema.literal/param-value+ [:sequential ::lib.schema.literal/param-value]]]][:row {:optional true} [:ref ::context.row]][:dimensions {:optional true} [:maybe [:ref ::context.row]]][:card-id {:optional true} [:maybe ::lib.schema.id/card]]])diff --git a/src/metabase/lib/schema/expression.cljc b/src/metabase/lib/schema/expression.cljcindex 4841126feac..5daeb1f611f 100644--- a/src/metabase/lib/schema/expression.cljc+++ b/src/metabase/lib/schema/expression.cljc@@ -4,7 +4,6 @@[malli.core :as mc][metabase.lib.dispatch :as lib.dispatch][metabase.lib.hierarchy :as lib.hierarchy]- [metabase.lib.options :as lib.options][metabase.lib.schema.common :as common][metabase.types.core :as types][metabase.util :as u]@@ -44,7 +43,7 @@[false [:ref ::base-type]]]"Determine the type of an MBQL expression. Returns either a type keyword, or if the type is ambiguous, a set ofpossible types."- [expr]+ [expr :- [:ref ::common/clause-arg]](or;; for MBQL clauses with `:effective-type` or `:base-type` in their options: ignore their dumb [[type-of-method]] methods;; and return that type directly. Ignore everything else! Life hack!@@ -260,7 +259,7 @@{:error/message "expressions must have unique names"}(fn [expressions](or (empty? expressions)- (apply distinct? (map #(:lib/expression-name (lib.options/options %)) expressions))))]])+ (apply distinct? (map (fn [[_tag opts]] (:lib/expression-name opts)) expressions))))]])(mr/def ::positive-integer-or-numeric-expression[:anddiff --git a/src/metabase/lib/schema/extraction.cljc b/src/metabase/lib/schema/extraction.cljcindex 4f1a3b4fc09..4f9f65d7863 100644--- a/src/metabase/lib/schema/extraction.cljc+++ b/src/metabase/lib/schema/extraction.cljc@@ -4,7 +4,7 @@[metabase.util.malli.registry :as mr]))(mr/def ::extraction- [:map+ [:map {:closed true}[:lib/type [:= :metabase.lib.extraction/extraction]][:tag [:enum:domain :subdomain :host :pathdiff --git a/src/metabase/lib/schema/filter.cljc b/src/metabase/lib/schema/filter.cljcindex a34618c5d41..b05be506109 100644--- a/src/metabase/lib/schema/filter.cljc+++ b/src/metabase/lib/schema/filter.cljc@@ -46,7 +46,7 @@(mr/def ::string-filter-options"String filter operator options. Only set for `:contains`, `:does-not-contain`, `:starts-with`, `:ends-with`operators."- [:map [:case-sensitive {:optional true} :boolean]]) ; default true+ [:map {:closed true} [:case-sensitive {:optional true} :boolean]]) ; default true(mr/def ::number-filter-operator"Numeric filter operators supported by the FE."@@ -79,7 +79,7 @@(mr/def ::time-interval-options"Options for `:time-interval` operator. Note that `:relative-time-interval` does not support these options."- [:map [:include-current {:optional true} :boolean]]) ; default false+ [:map {:closed true} [:include-current {:optional true} :boolean]]) ; default false(doseq [op [:and :or]](mbql-clause/define-catn-mbql-clause op :- :type/Boolean@@ -192,7 +192,7 @@[false ::id/segment]]])(mr/def ::operator- [:map+ [:map {:closed true}[:lib/type [:= :operator/filter]][:short [:enum:= :!= :inside :between :< :> :<= :>= :is-null :not-null :is-empty :not-emptydiff --git a/src/metabase/lib/schema/join.cljc b/src/metabase/lib/schema/join.cljcindex 13c2e96025d..a1c529fbce3 100644--- a/src/metabase/lib/schema/join.cljc+++ b/src/metabase/lib/schema/join.cljc@@ -165,7 +165,7 @@true))]])(mr/def ::strategy.option- [:map+ [:map {:closed true}[:lib/type [:= :option/join.strategy]][:strategy [:ref ::strategy]][:default {:optional true} :boolean]])diff --git a/src/metabase/lib/schema/mbql_clause.cljc b/src/metabase/lib/schema/mbql_clause.cljcindex dce3984c064..85dbd6c4a8f 100644--- a/src/metabase/lib/schema/mbql_clause.cljc+++ b/src/metabase/lib/schema/mbql_clause.cljc@@ -32,9 +32,15 @@(def ^:private invalid-clause-schema[:fn {:error/message "not a known MBQL clause"} (constantly false)])+(mr/def ::schema-form+ "A Malli schema, in its unparsed (vector/keyword/registry-name) form."+ [:fn+ {:description "a malli schema form"}+ #(or (keyword? %) (mc/schema? %) (and (vector? %) (keyword? (first %))))])+(mu/defn- mbql-clause-tag :- [:maybe :keyword]"If `x` is a (possibly not-yet-normalized) MBQL clause, return its `tag`."- [x]+ [x :- ::common/clause-tag-candidate](when (and (sequential? x)((some-fn keyword? string?) (first x)))(keyword (first x))))@@ -81,7 +87,7 @@::common/options[:ref :metabase.lib.schema.expression/expression]])"([tag :- keyword?- schema]+ schema :- ::schema-form](let [schema-name (tag->registered-schema-name tag)](mr/def schema-name schema);; only need to update the registry and calculated schemas if this is the very first time we're defining this@@ -94,7 +100,7 @@([tag :- keyword?_arrow :- [:= :-]return-type :- ::expression/base-type- schema]+ schema :- ::schema-form](define-mbql-clause tag schema)(defmethod expression/type-of-method tag[_clause]diff --git a/src/metabase/lib/schema/metadata.cljc b/src/metabase/lib/schema/metadata.cljcindex 7287ecbdc01..f87860443f9 100644--- a/src/metabase/lib/schema/metadata.cljc+++ b/src/metabase/lib/schema/metadata.cljc@@ -335,16 +335,10 @@[:enum {:decode/normalize keyword} :aggregation :fields :breakout :native])(mr/def ::column.snapshot- "A whole column metadata map stashed on *another* column under one of the internal keys below.-- Deliberately NOT `[:ref ::column]`: [[malli.util/merge]] recurses into the entries that the two schemas being merged- share, so a self-referential `::column` entry makes any `[:merge ...]` of two column-bearing maps -- for instance- `:metabase.lib.schema.drill-thru/drill-thru.column-filter`, which merges two maps that both have a `:column` --- recur until the stack blows. This shallow shape says what the value is without reintroducing the cycle."- [:map- [:lib/type [:= {:decode/normalize lib.schema.common/normalize-keyword} :metadata/column]]- [:name :string]- [:base-type {:optional true} ::lib.schema.common/base-type]])+ "A whole column metadata map stashed on *another* column under one of the internal keys below: the same shape as+ [[::column]] minus the `:and`-level constraints, so validating the stashed copy does not re-run those constraints+ against the column it was copied from."+ [:ref ::column.map])(mr/def ::column.timestamp"A timestamp column of a `metabase_field` row: a `java.time` value on the JVM, the string it was encoded as elsewhere."@@ -389,6 +383,301 @@[:max-value {:optional true} [:maybe number?]][:max_value {:optional true} [:maybe number?]]])+(def ^:private column-map+ [:map+ {:closed true+ :error/message "Valid column metadata"+ :decode/normalize normalize-column+ :decode/api lib.schema.common/remove-internal-keys+ :encode/serialize lib.schema.common/remove-internal-keys}+ [:lib/type [:= {:decode/normalize lib.schema.common/normalize-keyword, :default :metadata/column} :metadata/column]]+ ;;+ ;; TODO (Cam 6/19/25) -- change all these comments to proper `:description`s like we have+ ;; in [[metabase.legacy-mbql.schema]] so we can generate this documentation from this schema or whatever.+ ;;+ ;; column names are allowed to be empty strings in SQL Server :/+ ;;+ ;; In almost EVERY circumstance you should try to avoid using `:name`, because it's not well-defined whether it's+ ;; the `:lib/original-name` or `:lib/deduplicated-name`, and it might be either one depending on where the metadata+ ;; came from. Prefer one of the other name keys instead, only falling back to `:name` if they are not present.+ [:name :string]+ ;; TODO -- ignore `base_type` and make `effective_type` required; see #29707+ [:base-type {:default :type/*} ::lib.schema.common/base-type]+ ;; This is nillable because internal remap columns have `:id nil`.+ [:id {:optional true} [:maybe ::lib.schema.id/field]]+ [:display-name {:optional true} [:maybe :string]]+ [:effective-type {:optional true} [:maybe ::lib.schema.common/base-type]]+ [:semantic-type {:optional true} [:maybe ::lib.schema.common/semantic-or-relation-type]]+ ;; type of this column in the data warehouse, e.g. `TEXT` or `INTEGER`+ [:database-type {:optional true} [:maybe :string]]+ [:active {:optional true} :boolean]+ [:visibility-type {:optional true} [:maybe ::column.visibility-type]]+ [:data-sensitivity {:optional true} [:maybe ::column.data-sensitivity]]+ ;; if this is a field from another table (implicit join), this is the field in the current table that should be+ ;; used to perform the implicit join. e.g. if current table is `VENUES` and this field is `CATEGORIES.ID`, then the+ ;; `fk_field_id` would be `VENUES.CATEGORY_ID`. In a `:field` reference this is saved in the options map as+ ;; `:source-field`.+ [:fk-field-id {:optional true} [:maybe ::lib.schema.id/field]]+ ;; if this is a field from another table (implicit join), this is the name of the source field. It can be either a+ ;; `:lib/desired-column-alias` or `:name`, depending on the `:lib/source`. It's set only when the field can be+ ;; referenced by a name, normally when it's coming from a card or a previous query stage.+ [:fk-field-name {:optional true} [:maybe :string]]+ ;; if this is a field from another table (implicit join), this is the join alias of the source field.+ [:fk-join-alias {:optional true} [:maybe ::lib.schema.join/alias]]+ ;; `metabase_field.fk_target_field_id` in the application database; recorded during the sync process. This Field is+ ;; an foreign key, and points to this Field ID. This is mostly used to determine how to add implicit joins by+ ;; the [[metabase.query-processor.middleware.add-implicit-joins]] middleware.+ [:fk-target-field-id {:optional true} [:maybe ::lib.schema.id/field]]+ ;; Join alias of the table we're joining against, if any. SHOULD ONLY BE SET IF THE JOIN HAPPENED AT THIS STAGE OF+ ;; THE QUERY! (Also ok within a join's conditions for previous joins within the parent stage, because a join is+ ;; allowed to join on the results of something else)+ [:lib/join-alias {:optional true} [:maybe ::lib.schema.join/alias]]+ ;; the initial join alias used when this column was first introduced; should be propagated even if the join was+ ;; from a previous stage.+ ;;+ ;; What about when the column comes from join `X`, but inside `X` itself it comes from join `Y`? I think in this+ ;; case we want the outside world to see `X` since `Y` is not visible outside of `X`.+ ;;+ ;; original join alias = X+ ;; column => [join X => join Y]+ ;;+ ;; It is not currently well-defined whether this appears when the join was the current stage or not, i.e. if it's+ ;; equal to `:lib/join-alias` when it is set or if it is only set if the join happened in a previous+ ;; stage, i.e. if it's `nil` when `:lib/join-alias` is set. It seems like current behavior is the+ ;; former but you should NOT rely on this behavior.+ [:lib/original-join-alias {:optional true} [:maybe ::lib.schema.join/alias]]+ ;; these should only be present if temporal bucketing or binning is done in the current stage of the query; if+ ;; this happened in a previous stage they should get propagated as the keys below instead.+ [:lib/temporal-unit {:optional true} [:maybe ::lib.schema.temporal-bucketing/unit]]+ [:lib/binning {:optional true} [:maybe ::lib.schema.binning/binning]]+ ;; For nested fields (fields with a `:parent-id`, e.g. JSON columns), the pre-computed display name including the+ ;; full parent chain prefix, e.g. `"Grandparent: Parent: Child"`. Used as the highest-priority initial display+ ;; name in the display name pipeline, before join alias, binning, and temporal bucketing decorations are added.+ ;; This is distinct from `:lib/original-display-name`, which for a nested field stores just the leaf name+ ;; (e.g. `"Child"`) — both may coexist on the same column.+ [:lib/simple-display-name {:optional true} [:maybe ::lib.schema.common/non-blank-string]]+ [:lib/original-effective-type {:optional true} [:maybe ::lib.schema.common/base-type]]+ [:lib/transformation-added-base-type {:optional true} [:maybe :boolean]]+ ;; If temporal bucketing or binning happened in a previous stage, they are propagated as the keys below.+ ;; `:inherited-temporal-unit` signals that this column was already bucketed upstream, so the default temporal+ ;; unit becomes `:inherited` rather than a type-based default like `:month`, preventing double-bucketing.+ [:inherited-temporal-unit {:optional true} [:maybe ::lib.schema.temporal-bucketing/unit]]+ [:lib/original-binning {:optional true} [:maybe ::lib.schema.binning/binning]]+ ;; name of the expression where this column metadata came from. Should only be included for expressions introduced+ ;; at THIS STAGE of the query. If it's included elsewhere, that's an error. Thus this is the definitive way to know+ ;; if a column is "custom" in this stage (needs an `:expression` reference) or not.+ [:lib/expression-name {:optional true} [:maybe ::lib.schema.common/non-blank-string]]+ ;;+ ;; the name of the expression where this column came from, if the column came from a previous stage of the query+ [:lib/original-expression-name {:optional true} [:maybe ::lib.schema.common/non-blank-string]]+ ;; where this column came from. See docstring for `::column.source`.+ [:lib/source {:optional true} [:maybe [:ref ::column.source]]]+ ;;+ ;; if this column metadata was calculated based on MBQL clauses in the query itself, this is the UUID of the+ ;; clauses in question. Required for aggregations and expressions+ [:lib/source-uuid {:optional true} [:maybe [:ref ::lib.schema.common/uuid]]]+ ;;+ ;; whether this column metadata occurs in the `:breakout`(s) in the CURRENT STAGE or not. Previously this was+ ;; signified by `:lib/source = :source/breakouts` (which has been removed)+ ;;+ ;; this SHOULD NOT get propagated to subsequent stages!+ [:lib/breakout? {:optional true} [:maybe :boolean]]+ ;;+ ;; ID of the Card this came from, if this column originally came from a Card (Saved Question or Model). Mostly+ ;; used for creating column groups. AFAIK this should get propagated indefinitely -- Cam+ [:lib/card-id {:optional true} [:maybe ::lib.schema.id/card]]+ ;;+ ;; Whether this column originally was introduced by a Model. Model metadata has special rules, for example because+ ;; `:display-name` can be user-edited we should be careful not to recalculate it unnecessarily. See for+ ;; example [[metabase.query-processor.model-test/preserve-model-display-names-test]].+ ;;+ ;; This key should get propagated indefinitely.+ [:lib/from-model? {:optional true} [:maybe :boolean]]+ ;;+ ;; this stuff is adapted from [[metabase.query-processor.util.add-alias-info]]. It is included in+ ;; the [[metabase.lib.metadata.calculation/metadata]]+ ;;+ ;; the alias that should be used to this clause on the LHS of a `SELECT <lhs> AS <rhs>` or equivalent, i.e. the+ ;; name of this clause as exported by the previous stage, source table, or join.+ [:lib/source-column-alias {:optional true} [:maybe ::source-column-alias]]+ ;; the name we should export this column as, i.e. the RHS of a `SELECT <lhs> AS <rhs>` or equivalent. This is+ ;; guaranteed to be unique in each stage of the query.+ [:lib/desired-column-alias {:optional true} [:maybe ::desired-column-alias]]+ ;;+ ;; see description in schemas above+ ;;+ [:lib/original-name {:optional true} ::original-name]+ [:lib/deduplicated-name {:optional true} ::deduplicated-name]+ ;;+ ;; the original display name of this column before adding join/temporal bucketing/binning stuff to it. `Join ->+ ;; <whatever>` or `<whatever>: Month` or `<whatever>: Auto-binned`. Should be equal to the very first+ ;; `:display-name` we see when the column comes out of a metadata provider. Usually this is auto-generated with+ ;; humanized names from `:name`, but may differ.+ ;;+ ;; For model columns, this is set to the clean column name from the model's result_metadata with any+ ;; join arrow prefix stripped (e.g. "Products → Category" becomes "Category").+ [:lib/original-display-name {:optional true} [:maybe :string]]+ ;;+ ;; If this metadata was resolved from a ref (e.g. a `:field` ref) that contained a `:display-name` in the options,+ ;; this is that display name. `:lib/ref-display-name` should override any display names specified in the metadata.+ [:lib/ref-display-name {:optional true} [:maybe :string]]+ ;;+ ;; If this metadata was resolved from a ref (e.g. a `:field` ref) and that ref contained a `:name` in the options,+ ;; this is that name. If specified we should use this as the basis for the desired column alias rather than join+ ;; alias + source column alias.+ [:lib/ref-name {:optional true} [:maybe :string]]+ ;;+ ;; when column metadata is returned by certain things+ ;; like [[metabase.lib.aggregation/selected-aggregation-operators]] or [[metabase.lib.field/fieldable-columns]], it+ ;; might include this key, which tells you whether or not that column is currently selected or not already, e.g.+ ;; for [[metabase.lib.field/fieldable-columns]] it means its already present in `:fields`+ [:selected? {:optional true} :boolean]+ ;;+ ;; REMAPPING & FIELD VALUES+ ;;+ ;; See notes above for more info. `:has-field-values` comes from the application database and is used to decide+ ;; whether to sync FieldValues when running sync, and what certain FE QB widgets should+ ;; do. (See [[metabase.lib.field/field-values-search-info]]). Note that all metadata providers may not return this+ ;; column. The JVM provider currently does not, since the QP doesn't need it for anything.+ [:has-field-values {:optional true} [:maybe [:ref ::column.has-field-values]]]+ ;;+ ;; info about stuff like min and max values of the column, used for auto bucketing.+ [:fingerprint {:optional true} [:maybe [:ref ::lib.schema.metadata.fingerprint/fingerprint]]]+ ;;+ ;; If this is a nested column, the names of the ancestor columns used to access it. E.g. if the column is+ ;; `grandparent.parent.child` then `:nfc-path` would be `["grandparent" "parent"]`.+ ;;+ ;; This was originally added to power Postgres JSON column support; but is now used for any sort of nested column,+ ;; including BigQuery `RECORD` columns and MongoDB nested columns. See+ ;; https://metaboat.slack.com/archives/C0645JP1W81/p1754949404592539 for code archeology+ [:nfc-path {:optional true} [:maybe [:sequential :string]]]+ ;; the rest of the `metabase_field` row a metadata provider hands back+ [:table-id {:optional true} [:maybe [:or ::lib.schema.id/table [:re #"^card__\d+$"]]]]+ [:parent-id {:optional true} [:maybe ::lib.schema.id/field]]+ [:description {:optional true} [:maybe :string]]+ [:caveats {:optional true} [:maybe :string]]+ [:points-of-interest {:optional true} [:maybe :string]]+ [:coercion-strategy {:optional true} [:maybe ::lib.schema.common/coercion-strategy]]+ [:position {:optional true} [:maybe :int]]+ [:custom-position {:optional true} [:maybe :int]]+ [:database-position {:optional true} [:maybe :int]]+ [:database-default {:optional true} [:maybe :string]]+ [:database-indexed {:optional true} [:maybe :boolean]]+ [:database-is-auto-increment {:optional true} [:maybe :boolean]]+ [:database-is-generated {:optional true} [:maybe :boolean]]+ [:database-is-nullable {:optional true} [:maybe :boolean]]+ [:database-is-pk {:optional true} [:maybe :boolean]]+ [:database-partitioned {:optional true} [:maybe :boolean]]+ [:database-required {:optional true} [:maybe :boolean]]+ [:json-unfolding {:optional true} [:maybe :boolean]]+ [:preview-display {:optional true} [:maybe :boolean]]+ [:fingerprint-version {:optional true} [:maybe :int]]+ [:created-at {:optional true} [:maybe [:ref ::column.timestamp]]]+ [:updated-at {:optional true} [:maybe [:ref ::column.timestamp]]]+ [:last-analyzed {:optional true} [:maybe [:ref ::column.timestamp]]]+ ;; the Field a foreign key points at, and the Field that names this table's rows: hydrated onto a Field by the+ ;; application database metadata provider, but only an id reference in the metadata the frontend hands over+ [:target {:optional true} [:maybe [:or ::lib.schema.id/field [:ref ::column]]]]+ [:name-field {:optional true} [:maybe [:or ::lib.schema.id/field [:ref ::column]]]]+ ;; the Dimension and FieldValues joined onto a Field by the application database metadata provider+ [:dimension/id {:optional true} [:maybe ::lib.schema.id/dimension]]+ [:dimension/name {:optional true} [:maybe :string]]+ [:dimension/type {:optional true} [:maybe [:enum {:decode/normalize lib.schema.common/normalize-keyword} :internal :external]]]+ [:dimension/human-readable-field-id {:optional true} [:maybe ::lib.schema.id/field]]+ [:values/values {:optional true} [:maybe [:sequential [:ref ::lib.schema.literal/literal]]]]+ [:values/human-readable-values {:optional true} [:maybe [:sequential [:ref ::lib.schema.literal/literal]]]]+ [:field-values {:optional true} [:maybe [:sequential ::lib.schema.common/field-value]]]+ ;; what Lib annotates a column with when it returns it from a query+ [:breakout-positions {:optional true} [:maybe [:sequential :int]]]+ [:filter-positions {:optional true} [:maybe [:sequential :int]]]+ [:order-by-position {:optional true} [:maybe :int]]+ [:converted-timezone {:optional true} [:maybe :string]]+ [:remapped-from {:optional true} [:maybe :string]]+ [:remapped-to {:optional true} [:maybe :string]]+ [:remapped-from-index {:optional true} [:maybe :int]]+ [:remapping {:optional true} [:maybe [:map-of [:ref ::lib.schema.literal/literal] [:ref ::lib.schema.literal/literal]]]]+ [:dimension-interestingness {:optional true} [:maybe number?]]+ [:options {:optional true} [:maybe [:ref ::column.options]]]+ [:lib/options {:optional true} [:maybe [:ref ::column.lib-options]]]+ [:lib/original-fk-field-id {:optional true} [:maybe ::lib.schema.id/field]]+ [:lib/original-fk-field-name {:optional true} [:maybe :string]]+ [:lib/original-fk-join-alias {:optional true} [:maybe ::lib.schema.join/alias]]+ [:lib/original-join-name {:optional true} [:maybe :string]]+ [:lib/source-display-name {:optional true} [:maybe :string]]+ [:metabase.lib.join/HACK-from-incomplete-join? {:optional true} [:maybe :boolean]]+ ;; where the Mongo driver keeps the aliases of a `:field` ref while it compiles the ref as a column+ [:metabase.driver.mongo.query-processor/source-alias {:optional true} [:maybe :string]]+ [:metabase.driver.mongo.query-processor/join-field {:optional true} [:maybe :string]]+ [:metabase.driver.mongo.query-processor/inherited? {:optional true} [:maybe :boolean]]+ ;;+ ;; `:unit` and `:binning-info` are legacy keys and should not be set on a Lib column; `:lib/temporal-unit` and+ ;; `:lib/binning` are the keys for that. [[metabase.lib.metadata.result-metadata]] pins these two on a column on its+ ;; way to becoming legacy result metadata, which is the only reason they are declared.+ [:unit {:optional true} [:maybe [:ref ::lib.schema.temporal-bucketing/unit]]]+ [:binning-info {:optional true} [:maybe [:ref ::column.binning-info]]]+ ;;+ ;; populated by the `metabase_field.settings` column in the application database; I'm not really sure what goes in+ ;; here and if it's actually used for anything important in Lib or the QP (I suspect it's not).+ [:settings {:optional true} [:maybe [:ref ::lib.schema.common/visualization-settings]]]+ ;;+ ;; Added by [[metabase.lib.metadata.result-metadata]] primarily for legacy/backward-compatibility purposes with+ ;; legacy viz settings. This should not be used for anything other than that.+ [:metabase.lib.metadata.result-metadata/field-ref+ {:optional true}+ [:maybe #?(:cljs [:or+ [:ref :metabase.legacy-mbql.schema/Reference]+ [:fn {:error/message "JS array"}+ array?]]+ :clj [:ref :metabase.legacy-mbql.schema/Reference])]]+ ;;+ [:metabase.lib.metadata.result-metadata/source {:optional true} [:maybe [:ref ::column.legacy-source]]]+ ;;+ ;; these next two keys are derived by looking at `FieldValues` and `Dimension` instances associated with a `Field`;+ ;; they are used by the Query Processor to add column remappings to query results. To see how this maps to stuff in+ ;; the application database, look at the implementation for fetching a `:metadata/column`+ ;; in [[metabase.lib-be.metadata.jvm]]. I don't think this is really needed on the FE, at any rate the JS metadata+ ;; provider doesn't add these keys.+ [:lib/external-remap {:optional true} [:maybe [:ref ::column.remapping.external]]]+ [:lib/internal-remap {:optional true} [:maybe [:ref ::column.remapping.internal]]]+ ;;+ ;; The [[metabase.query-processor.middleware.add-implicit-clauses/add-implicit-fields]] middleware adds+ ;; `:qp/added-implicit-fields?` to stages where it adds implicit fields,+ ;; then [[metabase.lib.stage/fields-columns]] adds this key to any col from such a+ ;; stage. [[metabase.lib.metadata.result-metadata/super-broken-legacy-field-ref]] uses this to know to force Field+ ;; ID refs for QP `:field_ref` in results metadata to preserve historic behavior to avoid breaking legacy viz+ ;; settings that use it as a key.+ [:qp/implicit-field? {:optional true} [:maybe :boolean]]+ ;;+ ;; Coercion strategies (eg. UNIX seconds -> :type/DateTime) are configured on :model/Fields in the Admin UI, and+ ;; reflected on refs to that field on the *first* stage of an MBQL query or a join, where the column first appears+ ;; in the query. After the column passes through a stage boundary, it's no longer marked with the coercion strategy+ ;; to avoid double-coercion.+ ;;+ ;; *However*, when a table is sandboxed by a native query, and it has fields which need coercion, the SQL subquery+ ;; does not do the coercion (it's supposed to be a drop-in replacement for the table) but then the MBQL refs to its+ ;; columns are not in the first stage anymore! So the sandboxing middleware sets this flag to the+ ;; `:coercion-strategy`, along with `:qp/native-sandbox-column.propagate-coercion? true` (see below). Lib will+ ;; propagate the coercion strategy through *exactly one* stage boundary, so it can get from the SQL first stage to+ ;; the earliest MBQL stage, where the coercion will get applied correctly. See QUE2-376 or #69867 for more details.+ [:qp/native-sandbox-column.force-coercion-strategy {:optional true} [:ref ::lib.schema.common/coercion-strategy]]+ ;;+ ;; See above about `:qp/native-sandbox-column.force-coercion-strategy`.+ [:qp/native-sandbox-column.propagate-coercion? {:optional true} :boolean]+ [:metabase.lib.metadata.result-metadata/remove-join-alias? {:optional true} :boolean]+ [:lib/original-ref-style-for-result-metadata-purposes+ {:optional true}+ [:enum :original-ref-style/id :original-ref-style/name]]+ [:metabase.lib.field.resolution/fallback-metadata? {:optional true} :boolean]+ [:metabase.lib.join/target {:optional true} [:ref ::column.snapshot]]+ [:metabase.lib.underlying/original {:optional true} [:ref ::column.snapshot]]+ [:metabase.lib.underlying/temporal-unit {:optional true} [:maybe [:ref ::lib.schema.temporal-bucketing/unit]]]+ [:metabase.lib.underlying/binning {:optional true} [:maybe [:ref ::lib.schema.binning/binning]]]+ [:metabase.query-processor.pivot/idx {:optional true} [:int {:min 0}]]])++(mr/def ::column.map+ "The map portion of [[::column]], without its constraints."+ column-map)+(mr/def ::column"Malli schema for a valid map of column metadata, which can mean one of two things:@@ -403,294 +692,7 @@that they are largely compatible. So they're the same for now. We can revisit this in the future if we actually wantto differentiate between the two versions."[:and- [:map- {:closed true- :error/message "Valid column metadata"- :decode/normalize normalize-column- :decode/api lib.schema.common/remove-internal-keys- :encode/serialize lib.schema.common/remove-internal-keys}- [:lib/type [:= {:decode/normalize lib.schema.common/normalize-keyword, :default :metadata/column} :metadata/column]]- ;;- ;; TODO (Cam 6/19/25) -- change all these comments to proper `:description`s like we have- ;; in [[metabase.legacy-mbql.schema]] so we can generate this documentation from this schema or whatever.- ;;- ;; column names are allowed to be empty strings in SQL Server :/- ;;- ;; In almost EVERY circumstance you should try to avoid using `:name`, because it's not well-defined whether it's- ;; the `:lib/original-name` or `:lib/deduplicated-name`, and it might be either one depending on where the metadata- ;; came from. Prefer one of the other name keys instead, only falling back to `:name` if they are not present.- [:name :string]- ;; TODO -- ignore `base_type` and make `effective_type` required; see #29707- [:base-type {:default :type/*} ::lib.schema.common/base-type]- ;; This is nillable because internal remap columns have `:id nil`.- [:id {:optional true} [:maybe ::lib.schema.id/field]]- [:display-name {:optional true} [:maybe :string]]- [:effective-type {:optional true} [:maybe ::lib.schema.common/base-type]]- [:semantic-type {:optional true} [:maybe ::lib.schema.common/semantic-or-relation-type]]- ;; type of this column in the data warehouse, e.g. `TEXT` or `INTEGER`- [:database-type {:optional true} [:maybe :string]]- [:active {:optional true} :boolean]- [:visibility-type {:optional true} [:maybe ::column.visibility-type]]- [:data-sensitivity {:optional true} [:maybe ::column.data-sensitivity]]- ;; if this is a field from another table (implicit join), this is the field in the current table that should be- ;; used to perform the implicit join. e.g. if current table is `VENUES` and this field is `CATEGORIES.ID`, then the- ;; `fk_field_id` would be `VENUES.CATEGORY_ID`. In a `:field` reference this is saved in the options map as- ;; `:source-field`.- [:fk-field-id {:optional true} [:maybe ::lib.schema.id/field]]- ;; if this is a field from another table (implicit join), this is the name of the source field. It can be either a- ;; `:lib/desired-column-alias` or `:name`, depending on the `:lib/source`. It's set only when the field can be- ;; referenced by a name, normally when it's coming from a card or a previous query stage.- [:fk-field-name {:optional true} [:maybe :string]]- ;; if this is a field from another table (implicit join), this is the join alias of the source field.- [:fk-join-alias {:optional true} [:maybe ::lib.schema.join/alias]]- ;; `metabase_field.fk_target_field_id` in the application database; recorded during the sync process. This Field is- ;; an foreign key, and points to this Field ID. This is mostly used to determine how to add implicit joins by- ;; the [[metabase.query-processor.middleware.add-implicit-joins]] middleware.- [:fk-target-field-id {:optional true} [:maybe ::lib.schema.id/field]]- ;; Join alias of the table we're joining against, if any. SHOULD ONLY BE SET IF THE JOIN HAPPENED AT THIS STAGE OF- ;; THE QUERY! (Also ok within a join's conditions for previous joins within the parent stage, because a join is- ;; allowed to join on the results of something else)- [:lib/join-alias {:optional true} [:maybe ::lib.schema.join/alias]]- ;; the initial join alias used when this column was first introduced; should be propagated even if the join was- ;; from a previous stage.- ;;- ;; What about when the column comes from join `X`, but inside `X` itself it comes from join `Y`? I think in this- ;; case we want the outside world to see `X` since `Y` is not visible outside of `X`.- ;;- ;; original join alias = X- ;; column => [join X => join Y]- ;;- ;; It is not currently well-defined whether this appears when the join was the current stage or not, i.e. if it's- ;; equal to `:lib/join-alias` when it is set or if it is only set if the join happened in a previous- ;; stage, i.e. if it's `nil` when `:lib/join-alias` is set. It seems like current behavior is the- ;; former but you should NOT rely on this behavior.- [:lib/original-join-alias {:optional true} [:maybe ::lib.schema.join/alias]]- ;; these should only be present if temporal bucketing or binning is done in the current stage of the query; if- ;; this happened in a previous stage they should get propagated as the keys below instead.- [:lib/temporal-unit {:optional true} [:maybe ::lib.schema.temporal-bucketing/unit]]- [:lib/binning {:optional true} [:maybe ::lib.schema.binning/binning]]- ;; For nested fields (fields with a `:parent-id`, e.g. JSON columns), the pre-computed display name including the- ;; full parent chain prefix, e.g. `"Grandparent: Parent: Child"`. Used as the highest-priority initial display- ;; name in the display name pipeline, before join alias, binning, and temporal bucketing decorations are added.- ;; This is distinct from `:lib/original-display-name`, which for a nested field stores just the leaf name- ;; (e.g. `"Child"`) — both may coexist on the same column.- [:lib/simple-display-name {:optional true} [:maybe ::lib.schema.common/non-blank-string]]- [:lib/original-effective-type {:optional true} [:maybe ::lib.schema.common/base-type]]- [:lib/transformation-added-base-type {:optional true} [:maybe :boolean]]- ;; If temporal bucketing or binning happened in a previous stage, they are propagated as the keys below.- ;; `:inherited-temporal-unit` signals that this column was already bucketed upstream, so the default temporal- ;; unit becomes `:inherited` rather than a type-based default like `:month`, preventing double-bucketing.- [:inherited-temporal-unit {:optional true} [:maybe ::lib.schema.temporal-bucketing/unit]]- [:lib/original-binning {:optional true} [:maybe ::lib.schema.binning/binning]]- ;; name of the expression where this column metadata came from. Should only be included for expressions introduced- ;; at THIS STAGE of the query. If it's included elsewhere, that's an error. Thus this is the definitive way to know- ;; if a column is "custom" in this stage (needs an `:expression` reference) or not.- [:lib/expression-name {:optional true} [:maybe ::lib.schema.common/non-blank-string]]- ;;- ;; the name of the expression where this column came from, if the column came from a previous stage of the query- [:lib/original-expression-name {:optional true} [:maybe ::lib.schema.common/non-blank-string]]- ;; where this column came from. See docstring for `::column.source`.- [:lib/source {:optional true} [:maybe [:ref ::column.source]]]- ;;- ;; if this column metadata was calculated based on MBQL clauses in the query itself, this is the UUID of the- ;; clauses in question. Required for aggregations and expressions- [:lib/source-uuid {:optional true} [:maybe [:ref ::lib.schema.common/uuid]]]- ;;- ;; whether this column metadata occurs in the `:breakout`(s) in the CURRENT STAGE or not. Previously this was- ;; signified by `:lib/source = :source/breakouts` (which has been removed)- ;;- ;; this SHOULD NOT get propagated to subsequent stages!- [:lib/breakout? {:optional true} [:maybe :boolean]]- ;;- ;; ID of the Card this came from, if this column originally came from a Card (Saved Question or Model). Mostly- ;; used for creating column groups. AFAIK this should get propagated indefinitely -- Cam- [:lib/card-id {:optional true} [:maybe ::lib.schema.id/card]]- ;;- ;; Whether this column originally was introduced by a Model. Model metadata has special rules, for example because- ;; `:display-name` can be user-edited we should be careful not to recalculate it unnecessarily. See for- ;; example [[metabase.query-processor.model-test/preserve-model-display-names-test]].- ;;- ;; This key should get propagated indefinitely.- [:lib/from-model? {:optional true} [:maybe :boolean]]- ;;- ;; this stuff is adapted from [[metabase.query-processor.util.add-alias-info]]. It is included in- ;; the [[metabase.lib.metadata.calculation/metadata]]- ;;- ;; the alias that should be used to this clause on the LHS of a `SELECT <lhs> AS <rhs>` or equivalent, i.e. the- ;; name of this clause as exported by the previous stage, source table, or join.- [:lib/source-column-alias {:optional true} [:maybe ::source-column-alias]]- ;; the name we should export this column as, i.e. the RHS of a `SELECT <lhs> AS <rhs>` or equivalent. This is- ;; guaranteed to be unique in each stage of the query.- [:lib/desired-column-alias {:optional true} [:maybe ::desired-column-alias]]- ;;- ;; see description in schemas above- ;;- [:lib/original-name {:optional true} ::original-name]- [:lib/deduplicated-name {:optional true} ::deduplicated-name]- ;;- ;; the original display name of this column before adding join/temporal bucketing/binning stuff to it. `Join ->- ;; <whatever>` or `<whatever>: Month` or `<whatever>: Auto-binned`. Should be equal to the very first- ;; `:display-name` we see when the column comes out of a metadata provider. Usually this is auto-generated with- ;; humanized names from `:name`, but may differ.- ;;- ;; For model columns, this is set to the clean column name from the model's result_metadata with any- ;; join arrow prefix stripped (e.g. "Products → Category" becomes "Category").- [:lib/original-display-name {:optional true} [:maybe :string]]- ;;- ;; If this metadata was resolved from a ref (e.g. a `:field` ref) that contained a `:display-name` in the options,- ;; this is that display name. `:lib/ref-display-name` should override any display names specified in the metadata.- [:lib/ref-display-name {:optional true} [:maybe :string]]- ;;- ;; If this metadata was resolved from a ref (e.g. a `:field` ref) and that ref contained a `:name` in the options,- ;; this is that name. If specified we should use this as the basis for the desired column alias rather than join- ;; alias + source column alias.- [:lib/ref-name {:optional true} [:maybe :string]]- ;;- ;; when column metadata is returned by certain things- ;; like [[metabase.lib.aggregation/selected-aggregation-operators]] or [[metabase.lib.field/fieldable-columns]], it- ;; might include this key, which tells you whether or not that column is currently selected or not already, e.g.- ;; for [[metabase.lib.field/fieldable-columns]] it means its already present in `:fields`- [:selected? {:optional true} :boolean]- ;;- ;; REMAPPING & FIELD VALUES- ;;- ;; See notes above for more info. `:has-field-values` comes from the application database and is used to decide- ;; whether to sync FieldValues when running sync, and what certain FE QB widgets should- ;; do. (See [[metabase.lib.field/field-values-search-info]]). Note that all metadata providers may not return this- ;; column. The JVM provider currently does not, since the QP doesn't need it for anything.- [:has-field-values {:optional true} [:maybe [:ref ::column.has-field-values]]]- ;;- ;; info about stuff like min and max values of the column, used for auto bucketing.- [:fingerprint {:optional true} [:maybe [:ref ::lib.schema.metadata.fingerprint/fingerprint]]]- ;;- ;; If this is a nested column, the names of the ancestor columns used to access it. E.g. if the column is- ;; `grandparent.parent.child` then `:nfc-path` would be `["grandparent" "parent"]`.- ;;- ;; This was originally added to power Postgres JSON column support; but is now used for any sort of nested column,- ;; including BigQuery `RECORD` columns and MongoDB nested columns. See- ;; https://metaboat.slack.com/archives/C0645JP1W81/p1754949404592539 for code archeology- [:nfc-path {:optional true} [:maybe [:sequential :string]]]- ;; the rest of the `metabase_field` row a metadata provider hands back- [:table-id {:optional true} [:maybe ::lib.schema.id/table]]- [:parent-id {:optional true} [:maybe ::lib.schema.id/field]]- [:description {:optional true} [:maybe :string]]- [:caveats {:optional true} [:maybe :string]]- [:points-of-interest {:optional true} [:maybe :string]]- [:coercion-strategy {:optional true} [:maybe ::lib.schema.common/coercion-strategy]]- [:position {:optional true} [:maybe :int]]- [:custom-position {:optional true} [:maybe :int]]- [:database-position {:optional true} [:maybe :int]]- [:database-default {:optional true} [:maybe :string]]- [:database-indexed {:optional true} [:maybe :boolean]]- [:database-is-auto-increment {:optional true} [:maybe :boolean]]- [:database-is-generated {:optional true} [:maybe :boolean]]- [:database-is-nullable {:optional true} [:maybe :boolean]]- [:database-is-pk {:optional true} [:maybe :boolean]]- [:database-partitioned {:optional true} [:maybe :boolean]]- [:database-required {:optional true} [:maybe :boolean]]- [:json-unfolding {:optional true} [:maybe :boolean]]- [:preview-display {:optional true} [:maybe :boolean]]- [:fingerprint-version {:optional true} [:maybe :int]]- [:created-at {:optional true} [:maybe [:ref ::column.timestamp]]]- [:updated-at {:optional true} [:maybe [:ref ::column.timestamp]]]- [:last-analyzed {:optional true} [:maybe [:ref ::column.timestamp]]]- ;; the Field a foreign key points at, and the Field that names this table's rows: hydrated onto a Field by the- ;; application database metadata provider, but only an id reference in the metadata the frontend hands over- [:target {:optional true} [:maybe [:or ::lib.schema.id/field [:ref ::column]]]]- [:name-field {:optional true} [:maybe [:or ::lib.schema.id/field [:ref ::column]]]]- ;; the Dimension and FieldValues joined onto a Field by the application database metadata provider- [:dimension/id {:optional true} [:maybe ::lib.schema.id/dimension]]- [:dimension/name {:optional true} [:maybe :string]]- [:dimension/type {:optional true} [:maybe [:enum {:decode/normalize lib.schema.common/normalize-keyword} :internal :external]]]- [:dimension/human-readable-field-id {:optional true} [:maybe ::lib.schema.id/field]]- [:values/values {:optional true} [:maybe [:sequential [:ref ::lib.schema.literal/literal]]]]- [:values/human-readable-values {:optional true} [:maybe [:sequential [:ref ::lib.schema.literal/literal]]]]- ;; what Lib annotates a column with when it returns it from a query- [:breakout-positions {:optional true} [:maybe [:sequential :int]]]- [:filter-positions {:optional true} [:maybe [:sequential :int]]]- [:order-by-position {:optional true} [:maybe :int]]- [:converted-timezone {:optional true} [:maybe :string]]- [:remapped-from {:optional true} [:maybe :string]]- [:remapped-to {:optional true} [:maybe :string]]- [:remapped-from-index {:optional true} [:maybe :int]]- [:remapping {:optional true} [:maybe [:map-of [:ref ::lib.schema.literal/literal] [:ref ::lib.schema.literal/literal]]]]- [:dimension-interestingness {:optional true} [:maybe number?]]- [:options {:optional true} [:maybe [:ref ::column.options]]]- [:lib/options {:optional true} [:maybe [:ref ::column.lib-options]]]- [:lib/original-fk-field-id {:optional true} [:maybe ::lib.schema.id/field]]- [:lib/original-fk-field-name {:optional true} [:maybe :string]]- [:lib/original-fk-join-alias {:optional true} [:maybe ::lib.schema.join/alias]]- [:lib/original-join-name {:optional true} [:maybe :string]]- [:lib/source-display-name {:optional true} [:maybe :string]]- [:metabase.lib.join/HACK-from-incomplete-join? {:optional true} [:maybe :boolean]]- ;; where the Mongo driver keeps the aliases of a `:field` ref while it compiles the ref as a column- [:metabase.driver.mongo.query-processor/source-alias {:optional true} [:maybe :string]]- [:metabase.driver.mongo.query-processor/join-field {:optional true} [:maybe :string]]- [:metabase.driver.mongo.query-processor/inherited? {:optional true} [:maybe :boolean]]- ;;- ;; `:unit` and `:binning-info` are legacy keys and should not be set on a Lib column; `:lib/temporal-unit` and- ;; `:lib/binning` are the keys for that. [[metabase.lib.metadata.result-metadata]] pins these two on a column on its- ;; way to becoming legacy result metadata, which is the only reason they are declared.- [:unit {:optional true} [:maybe [:ref ::lib.schema.temporal-bucketing/unit]]]- [:binning-info {:optional true} [:maybe [:ref ::column.binning-info]]]- ;;- ;; populated by the `metabase_field.settings` column in the application database; I'm not really sure what goes in- ;; here and if it's actually used for anything important in Lib or the QP (I suspect it's not).- [:settings {:optional true} [:maybe [:ref ::lib.schema.common/visualization-settings]]]- ;;- ;; Added by [[metabase.lib.metadata.result-metadata]] primarily for legacy/backward-compatibility purposes with- ;; legacy viz settings. This should not be used for anything other than that.- [:metabase.lib.metadata.result-metadata/field-ref- {:optional true}- [:maybe #?(:cljs [:or- [:ref :metabase.legacy-mbql.schema/Reference]- [:fn {:error/message "JS array"}- array?]]- :clj [:ref :metabase.legacy-mbql.schema/Reference])]]- ;;- [:metabase.lib.metadata.result-metadata/source {:optional true} [:maybe [:ref ::column.legacy-source]]]- ;;- ;; these next two keys are derived by looking at `FieldValues` and `Dimension` instances associated with a `Field`;- ;; they are used by the Query Processor to add column remappings to query results. To see how this maps to stuff in- ;; the application database, look at the implementation for fetching a `:metadata/column`- ;; in [[metabase.lib-be.metadata.jvm]]. I don't think this is really needed on the FE, at any rate the JS metadata- ;; provider doesn't add these keys.- [:lib/external-remap {:optional true} [:maybe [:ref ::column.remapping.external]]]- [:lib/internal-remap {:optional true} [:maybe [:ref ::column.remapping.internal]]]- ;;- ;; The [[metabase.query-processor.middleware.add-implicit-clauses/add-implicit-fields]] middleware adds- ;; `:qp/added-implicit-fields?` to stages where it adds implicit fields,- ;; then [[metabase.lib.stage/fields-columns]] adds this key to any col from such a- ;; stage. [[metabase.lib.metadata.result-metadata/super-broken-legacy-field-ref]] uses this to know to force Field- ;; ID refs for QP `:field_ref` in results metadata to preserve historic behavior to avoid breaking legacy viz- ;; settings that use it as a key.- [:qp/implicit-field? {:optional true} [:maybe :boolean]]- ;;- ;; Coercion strategies (eg. UNIX seconds -> :type/DateTime) are configured on :model/Fields in the Admin UI, and- ;; reflected on refs to that field on the *first* stage of an MBQL query or a join, where the column first appears- ;; in the query. After the column passes through a stage boundary, it's no longer marked with the coercion strategy- ;; to avoid double-coercion.- ;;- ;; *However*, when a table is sandboxed by a native query, and it has fields which need coercion, the SQL subquery- ;; does not do the coercion (it's supposed to be a drop-in replacement for the table) but then the MBQL refs to its- ;; columns are not in the first stage anymore! So the sandboxing middleware sets this flag to the- ;; `:coercion-strategy`, along with `:qp/native-sandbox-column.propagate-coercion? true` (see below). Lib will- ;; propagate the coercion strategy through *exactly one* stage boundary, so it can get from the SQL first stage to- ;; the earliest MBQL stage, where the coercion will get applied correctly. See QUE2-376 or #69867 for more details.- [:qp/native-sandbox-column.force-coercion-strategy {:optional true} [:ref ::lib.schema.common/coercion-strategy]]- ;;- ;; See above about `:qp/native-sandbox-column.force-coercion-strategy`.- [:qp/native-sandbox-column.propagate-coercion? {:optional true} :boolean]- [:metabase.lib.metadata.result-metadata/remove-join-alias? {:optional true} :boolean]- [:lib/original-ref-style-for-result-metadata-purposes- {:optional true}- [:enum :original-ref-style/id :original-ref-style/name]]- [:metabase.lib.field.resolution/fallback-metadata? {:optional true} :boolean]- [:metabase.lib.join/target {:optional true} [:ref ::column.snapshot]]- [:metabase.lib.underlying/original {:optional true} [:ref ::column.snapshot]]- [:metabase.lib.underlying/temporal-unit {:optional true} [:maybe [:ref ::lib.schema.temporal-bucketing/unit]]]- [:metabase.lib.underlying/binning {:optional true} [:maybe [:ref ::lib.schema.binning/binning]]]- [:metabase.query-processor.pivot/idx {:optional true} [:int {:min 0}]]]+ column-map;;;; Additional constraints;;@@ -700,6 +702,7 @@(into {:binning ":binning is deprecated; use :lib/binning instead":field-ref ":field-ref is deprecated. For QP result metadata, use :metabase.lib.metadata.result-metadata/field-ref":ident ":ident is deprecated and should not be included in column metadata"+ :lib/model-display-name ":lib/model-display-name is deprecated and should not be included in column metadata":model/inner-ident ":model/inner_ident (normalized to :model/inner-ident) is deprecated and should not be included in column metadata":source ":source is deprecated; use :lib/source instead. For QP result metadata, use :metabase.lib.metadata.result-metadata/source":source-alias ":source-alias is deprecated; use :lib/join-alias or :lib/original-join-alias instead"@@ -710,17 +713,17 @@(mr/def ::persisted-info.definition"Definition spec for a cached table."- [:map+ [:map {:closed true}[:table-name ::lib.schema.common/non-blank-string][:field-definitions [:maybe [:sequential- [:map+ [:map {:closed true}[:field-name ::lib.schema.common/non-blank-string];; TODO check (isa? :type/Integer :type/*)[:base-type ::lib.schema.common/base-type]]]]]])(mr/def ::persisted-info"Persisted Info = Cached Table (?). See [[metabase.model-persistence.models.persisted-info]]"- [:map+ [:map {:closed true}[:active :boolean][:state ::lib.schema.common/non-blank-string][:table-name ::lib.schema.common/non-blank-string]@@ -839,7 +842,7 @@See [[metabase.lib.card/card-metadata-columns]] that converts these as needed."[:map- {:decode/normalize normalize-card+ {:closed true, :decode/normalize normalize-card:decode/mock mock-card:error/message "Valid Card metadata"}[:lib/type [:= :metadata/card]]@@ -858,6 +861,19 @@;; ID of the collection this Card is saved in. `nil` means it is saved in the "Root Collection". Important for;; perms-checking purposes.[:collection-id {:optional true} [:maybe ::lib.schema.id/collection]]+ [:entity-id {:optional true} [:maybe ::lib.schema.common/non-blank-string]]+ [:created-at {:optional true} [:maybe [:or :string #?(:clj (lib.schema.common/instance-of-class java.time.temporal.Temporal))]]]+ [:card-schema {:optional true} [:maybe :int]]+ [:visualization-settings {:optional true} [:maybe [:ref ::lib.schema.common/visualization-settings]]]+ [:description {:optional true} [:maybe :string]]+ [:display {:optional true} [:maybe [:or :keyword :string]]]+ [:archived {:optional true} [:maybe :boolean]]+ [:source-card-id {:optional true} [:maybe ::lib.schema.id/card]]+ [:dashboard-id {:optional true} [:maybe pos-int?]]+ [:dimensions {:optional true} [:maybe [:sequential [:ref ::persisted-dimension]]]]+ [:dimension-mappings {:optional true} [:maybe [:sequential [:ref ::dimension-mapping]]]]+ [:display-name {:optional true} [:maybe :string]]+ [:fields {:optional true} [:maybe [:ref ::card.result-metadata]]];;;; PERSISTED INFO: This comes from the [[metabase.model-persistence.models.persisted-info]] model.;;@@ -869,10 +885,21 @@(:id segment))(assoc :name (str "Segment " (:id segment)))))+(mr/def ::segment.definition+ "Segment definition: empty, an MBQL 5 query, or a legacy inner query."+ [:multi {:dispatch (fn [definition]+ (cond+ (empty? definition) :empty+ (:lib/type definition) :mbql5+ :else :legacy))}+ [:empty [:= {} {}]]+ [:mbql5 [:ref :metabase.lib.schema/query]]+ [:legacy [:ref :metabase.legacy-mbql.schema/MBQLInnerQuery]]])+(mr/def ::segment"More or less the same as a [[metabase.segments.models.segment]], but with kebab-case keys."[:map- {:error/message "Valid Segment metadata"+ {:closed true, :error/message "Valid Segment metadata":decode/mock mock-segment}[:lib/type [:= :metadata/segment]][:id ::lib.schema.id/segment]@@ -880,8 +907,11 @@[:table-id ::lib.schema.id/table];; the MBQL snippet defining this Segment; this may still be in legacy;; format. [[metabase.lib.segment/segment-definition]] handles conversion to MBQL 5 if needed.- [:definition [:maybe :map]]- [:description {:optional true} [:maybe ::lib.schema.common/non-blank-string]]])+ [:definition [:maybe [:ref ::segment.definition]]]+ [:description {:optional true} [:maybe ::lib.schema.common/non-blank-string]]+ [:entity-id {:optional true} [:maybe ::lib.schema.common/non-blank-string]]+ [:archived {:optional true} [:maybe :boolean]]+ [:filter-positions {:optional true} [:maybe [:sequential :int]]]])(defn- normalize-measure-definition [definition](when definition@@ -902,10 +932,74 @@(:id measure))(assoc :name (str "Measure " (:id measure)))))+(mr/def ::dimension-id+ "UUID string identifying a dimension."+ ::lib.schema.common/uuid)++(mr/def ::dimension-group+ "Group descriptor for a dimension, indicating which table it belongs to."+ [:map {:closed true}+ [:id :string]+ [:type [:enum "main" "connection"]]+ [:display-name :string]])++(mr/def ::dimension-source.type+ [:enum :field])++(mr/def ::dimension-source+ [:map {:closed true}+ [:type ::dimension-source.type]+ [:field-id {:optional true} [:maybe ::lib.schema.id/field]]+ [:binning {:optional true} [:maybe :boolean]]])++(mr/def ::dimension-mapping.type+ "Type of dimension mapping."+ [:enum :table])++(mr/def ::dimension-mapping.target+ "Target field reference for a dimension mapping, e.g. [:field {:source-field 1} 2]."+ [:ref :mbql.clause/field])++(mr/def ::dimension-mapping+ "Schema for a dimension mapping."+ [:map {:closed true}+ [:type ::dimension-mapping.type]+ [:table-id {:optional true} [:maybe ::lib.schema.id/table]]+ [:dimension-id ::dimension-id]+ [:target ::dimension-mapping.target]])++(mr/def ::dimension-status+ "Status of a dimension indicating whether it's active or has issues.+ - :status/active - Column exists, dimension is usable+ - :status/orphaned - Column was removed from schema, dimension preserved for reference"+ [:enum :status/active :status/orphaned])++(mr/def ::persisted-dimension+ "Schema for a persisted dimension definition with status tracking.+ Persisted dimensions include additional metadata about their status+ and any issues that prevent them from being used.+ Note: target field references are stored in dimension-mappings, not here."+ [:map {:closed true}+ [:id ::dimension-id]+ [:name {:optional true} [:maybe :string]]+ [:display-name {:optional true} [:maybe ::lib.schema.common/non-blank-string]]+ [:description {:optional true} [:maybe :string]]+ [:effective-type {:optional true} [:maybe ::lib.schema.common/base-type]]+ [:semantic-type {:optional true} [:maybe ::lib.schema.common/semantic-or-relation-type]]+ [:has-field-values {:optional true} [:maybe [:enum :list :search :none]]]+ [:status {:optional true} [:maybe ::dimension-status]]+ [:status-message {:optional true} [:maybe :string]]+ [:sources {:optional true} [:maybe [:sequential ::dimension-source]]]+ [:group {:optional true} [:maybe ::dimension-group]]+ [:lib/source {:optional true} [:maybe [:or [:ref ::column.source] :string]]]+ [:default-temporal-unit {:optional true} ::lib.schema.temporal-bucketing/unit]+ ;; At most one dimension per entity may be the default.+ [:default {:optional true} [:maybe :boolean]]])+(mr/def ::measure"More or less the same as a [[metabase.measures.models.measure]], but with kebab-case keys."[:map- {:error/message "Valid Measure metadata"+ {:closed true, :error/message "Valid Measure metadata":decode/mock mock-measure}[:lib/type [:= :metadata/measure]][:id ::lib.schema.id/measure]@@ -914,7 +1008,11 @@;; the MBQL snippet defining this Measure, contains an aggregation expression.;; Strict validation via ::lib.schema.measure/definition happens in metabase.measures.models.measure[:definition [:maybe [:ref ::measure.definition]]]- [:description {:optional true} [:maybe ::lib.schema.common/non-blank-string]]])+ [:description {:optional true} [:maybe ::lib.schema.common/non-blank-string]]+ [:archived {:optional true} [:maybe :boolean]]+ [:dimensions {:optional true} [:maybe [:sequential [:ref ::persisted-dimension]]]]+ [:dimension-mappings {:optional true} [:maybe [:sequential [:ref ::dimension-mapping]]]]+ [:aggregation-positions {:optional true} [:maybe [:sequential :int]]]])(mr/def ::metric"A V2 Metric! This a special subtype of a Card. Not convinced we really need this as opposed to just using `::card` --@@ -924,20 +1022,25 @@[:map[:lib/type [:= :metadata/metric]][:type [:= :metric]]- [:lib/join-alias {:optional true} ::lib.schema.common/non-blank-string]]])+ [:lib/join-alias {:optional true} ::lib.schema.common/non-blank-string]+ [:aggregation-position {:optional true} [:maybe [:int {:min 0}]]]]])(mr/def ::native-query-snippet- [:map+ [:map {:closed true}[:lib/type [:= :metadata/native-query-snippet]][:id ::lib.schema.id/native-query-snippet]+ [:name {:optional true} [:maybe :string]]+ [:description {:optional true} [:maybe :string]]+ [:content {:optional true} [:maybe :string]]+ [:archived {:optional true} [:maybe :boolean]]+ [:collection-id {:optional true} [:maybe ::lib.schema.id/collection]][:template-tags {:optional true} [:maybe [:ref ::lib.schema.template-tag/template-tag-map]]]])-;;; TODO (Cam 8/8/25) -- description, content, archived, collection-id(mr/def ::table"Schema for metadata about a specific [[metabase.warehouse-schema.models.table]]. More or less the same but withkebab-case keys."[:map- {:error/message "Valid Table metadata"}+ {:closed true, :error/message "Valid Table metadata"}[:lib/type [:= :metadata/table]][:id ::lib.schema.id/table][:name ::lib.schema.common/non-blank-string]@@ -946,24 +1049,90 @@;; Optional `:db` AST slot for cross-DB references (BigQuery `project.dataset.table`,;; SQL Server / Snowflake `db.schema.table`). Sync doesn't populate it on standard;; reads — only cross-DB rewriters fill it.- [:db {:optional true} [:maybe ::lib.schema.common/non-blank-string]]])+ [:db {:optional true} [:maybe ::lib.schema.common/non-blank-string]]+ [:db-id {:optional true} ::lib.schema.id/database]+ [:active {:optional true} [:maybe :boolean]]+ [:visibility-type {:optional true} [:maybe [:or :keyword :string]]]+ [:database-require-filter {:optional true} [:maybe :boolean]]+ [:description {:optional true} [:maybe :string]]+ [:entity-type {:optional true} [:maybe [:or :keyword :string]]]+ [:caveats {:optional true} [:maybe :string]]+ [:points-of-interest {:optional true} [:maybe :string]]+ [:field-order {:optional true} [:maybe [:or :keyword :string]]]+ [:initial-sync-status {:optional true} [:maybe [:or :string :keyword]]]+ [:show-in-getting-started {:optional true} [:maybe :boolean]]+ [:is-upload {:optional true} [:maybe :boolean]]+ [:entity-id {:optional true} [:maybe ::lib.schema.common/non-blank-string]]+ [:created-at {:optional true} [:maybe [:or :string #?(:clj (lib.schema.common/instance-of-class java.time.temporal.Temporal))]]]+ [:updated-at {:optional true} [:maybe [:or :string #?(:clj (lib.schema.common/instance-of-class java.time.temporal.Temporal))]]]+ [:deactivated-at {:optional true} [:maybe [:or :string #?(:clj (lib.schema.common/instance-of-class java.time.temporal.Temporal))]]]+ [:archived-at {:optional true} [:maybe [:or :string #?(:clj (lib.schema.common/instance-of-class java.time.temporal.Temporal))]]]+ [:estimated-row-count {:optional true} [:maybe :int]]+ [:view-count {:optional true} [:maybe :int]]+ [:is-writable {:optional true} [:maybe :boolean]]+ [:data-authority {:optional true} [:maybe [:or :keyword :string]]]+ [:data-source {:optional true} [:maybe [:or :keyword :string]]]+ [:data-layer {:optional true} [:maybe [:or :keyword :string]]]+ [:owner-email {:optional true} [:maybe :string]]+ [:owner-user-id {:optional true} [:maybe ::lib.schema.id/user]]+ [:collection-id {:optional true} [:maybe ::lib.schema.id/collection]]+ [:is-published {:optional true} [:maybe :boolean]]+ [:transform-id {:optional true} [:maybe ::lib.schema.id/transform]]+ [:transform-target {:optional true} [:maybe :boolean]]])(mr/def ::database"Malli schema for the DatabaseMetadata as returned by `GET /api/database/:id/metadata` -- what should be available tothe frontend Query Builder."[:map- {:error/message "Valid Database metadata"}+ {:closed true, :error/message "Valid Database metadata"}[:lib/type [:= :metadata/database]][:id ::lib.schema.id/database];; TODO -- this should validate against the driver features list in [[metabase.driver/features]] if we're in;; Clj mode- [:dbms-version {:optional true} [:maybe :map]]- [:details {:optional true} :map]+ [:dbms-version {:optional true} [:maybe [:map {:closed true}+ [:flavor {:optional true} [:maybe :string]]+ [:version {:optional true} [:maybe :string]]+ [:semantic-version {:optional true} [:maybe [:or+ [:sequential :int]+ [:map {:closed true} [:major :int] [:minor :int]]]]]+ [:cloud {:optional true} [:maybe :boolean]]]]]+ [:details {:optional true} ::lib.schema.common/database-details][:engine {:optional true} [:keyword {:decode/normalize lib.schema.common/normalize-keyword}]][:features {:optional true} [:set [:keyword {:decode/normalize lib.schema.common/normalize-keyword}]]][:is-audit {:optional true} :boolean][:is-attached-dwh {:optional true} :boolean]- [:settings {:optional true} [:maybe [:ref ::lib.schema.common/visualization-settings]]]])+ [:settings {:optional true} [:maybe [:ref ::lib.schema.common/visualization-settings]]]+ [:name {:optional true} [:maybe :string]]+ [:description {:optional true} [:maybe :string]]+ [:timezone {:optional true} [:maybe :string]]+ [:write-data-details {:optional true} [:maybe ::lib.schema.common/database-details]]+ [:admin-details {:optional true} [:maybe ::lib.schema.common/database-details]]+ [:router-database-id {:optional true} [:maybe ::lib.schema.id/database]]+ [:auto-run-queries {:optional true} [:maybe :boolean]]+ [:cache-field-values-schedule {:optional true} [:maybe :string]]+ [:metadata-sync-schedule {:optional true} [:maybe :string]]+ [:cache-ttl {:optional true} [:maybe :int]]+ [:caveats {:optional true} [:maybe :string]]+ [:points-of-interest {:optional true} [:maybe :string]]+ [:creator-id {:optional true} [:maybe pos-int?]]+ [:initial-sync-status {:optional true} [:maybe [:or :string :keyword]]]+ [:is-full-sync {:optional true} [:maybe :boolean]]+ [:is-on-demand {:optional true} [:maybe :boolean]]+ [:is-sample {:optional true} [:maybe :boolean]]+ [:native-permissions {:optional true} [:maybe [:or :keyword :string]]]+ [:options {:optional true} [:maybe [:ref ::lib.schema.common/database-settings]]]+ [:refingerprint {:optional true} [:maybe :boolean]]+ [:is-stub {:optional true} [:maybe :boolean]]+ [:provider-name {:optional true} [:maybe :string]]+ [:uploads-enabled {:optional true} [:maybe :boolean]]+ [:uploads-schema-name {:optional true} [:maybe :string]]+ [:uploads-table-prefix {:optional true} [:maybe :string]]+ [:can-manage {:optional true} [:maybe :boolean]]+ [:can-upload {:optional true} [:maybe :boolean]]+ [:router-user-attribute {:optional true} [:maybe :string]]+ [:transforms-permissions {:optional true} [:maybe [:or :keyword :string]]]+ [:created-at {:optional true} [:maybe [:or :string #?(:clj (lib.schema.common/instance-of-class java.time.temporal.Temporal))]]]+ [:updated-at {:optional true} [:maybe [:or :string #?(:clj (lib.schema.common/instance-of-class java.time.temporal.Temporal))]]]])(mr/def ::metadata-provider"Schema for something that satisfies the [[metabase.lib.metadata.protocols/MetadataProvider]] protocol."@@ -1007,9 +1176,57 @@[:lib/type [:= {:default :metadata/results} :metadata/results]][:columns [:sequential ::column]]])+(mr/def ::transform.source-incremental-strategy+ "Incremental strategy of a Transform source, as stored in the app DB."+ [:map {:closed true}+ [:type [:or :keyword :string]]+ [:checkpoint-filter-field-id {:optional true} [:maybe ::lib.schema.id/field]]+ [:lookback {:optional true} [:maybe [:map {:closed true}+ [:value pos-int?]+ [:unit :string]]]]])++(mr/def ::transform.target-incremental-strategy+ "Incremental strategy of a Transform target, as stored in the app DB."+ [:map {:closed true}+ [:type [:or :keyword :string]]+ [:unique-key {:optional true} [:sequential [:map {:closed true}+ [:name {:optional true} :string]+ [:field-id {:optional true} [:maybe ::lib.schema.id/field]]]]]])++(mr/def ::transform.source-table+ "One source table entry of a Python Transform source, as stored in the app DB."+ [:map {:closed true}+ [:alias :string]+ [:database_id {:optional true} [:maybe :int]]+ [:schema {:optional true} [:maybe :string]]+ [:table {:optional true} :string]+ [:table_id {:optional true} [:maybe :int]]])++(mr/def ::transform.source+ "The `:source` of a Transform: a query or a Python script."+ [:map {:closed true}+ [:type {:optional true} [:or :keyword :string]]+ [:query {:optional true} [:ref ::card.query]]+ [:source-incremental-strategy {:optional true} [:maybe [:ref ::transform.source-incremental-strategy]]]+ [:source-database {:optional true} [:maybe :int]]+ [:source-tables {:optional true} [:sequential [:ref ::transform.source-table]]]+ [:body {:optional true} :string]])++(mr/def ::transform.target+ "The `:target` table of a Transform."+ [:map {:closed true}+ [:type {:optional true} [:or :keyword :string]]+ [:database {:optional true} [:maybe ::lib.schema.id/database]]+ [:schema {:optional true} [:maybe :string]]+ [:name {:optional true} :string]+ [:target-incremental-strategy {:optional true} [:maybe [:ref ::transform.target-incremental-strategy]]]])+(mr/def ::transform"TODO (Cam 10/1/25) -- I'm putting this here as a placeholder until you guys go fill it out a little more."- [:map+ [:map {:closed true}+ [:lib/type [:= :metadata/transform]][:id ::lib.schema.id/transform]- [:source {:optional true} [:map- [:query {:optional true} [:ref :metabase.lib.schema/query]]]]])+ [:name {:optional true} :string]+ [:source {:optional true} [:ref ::transform.source]]+ [:source-type {:optional true} [:maybe [:enum :python :native :mbql]]]+ [:target {:optional true} [:ref ::transform.target]]])diff --git a/src/metabase/lib/schema/temporal_bucketing.cljc b/src/metabase/lib/schema/temporal_bucketing.cljcindex 795e16b3583..e52026babcb 100644--- a/src/metabase/lib/schema/temporal_bucketing.cljc+++ b/src/metabase/lib/schema/temporal_bucketing.cljc@@ -182,7 +182,8 @@datetime-interval-units))(mr/def ::option- [:map+ [:map {:closed true}[:lib/type [:= :option/temporal-bucketing]][:unit ::unit]- [:default {:optional true} :boolean]])+ [:default {:optional true} :boolean]+ [:selected {:optional true} :boolean]])diff --git a/src/metabase/lib/schema/test_spec.cljc b/src/metabase/lib/schema/test_spec.cljcindex c96ba5f3322..ca8cb487e5e 100644--- a/src/metabase/lib/schema/test_spec.cljc+++ b/src/metabase/lib/schema/test_spec.cljc@@ -13,12 +13,12 @@[metabase.util.malli.registry :as mr]))(mr/def ::test-table-source-spec- [:map+ [:map {:closed true}[:type [:= {:decode/normalize lib.schema.common/normalize-keyword} :table]][:id [:ref ::lib.schema.id/table]]])(mr/def ::test-card-source-spec- [:map+ [:map {:closed true}[:type [:= {:decode/normalize lib.schema.common/normalize-keyword} :card]][:id [:ref ::lib.schema.id/card]]])@@ -29,7 +29,7 @@[:card ::test-card-source-spec]])(mr/def ::test-column-spec- [:map+ [:map {:closed true}[:type [:= {:decode/normalize lib.schema.common/normalize-keyword} :column]][:name string?][:table-id {:optional true} [:maybe ::lib.schema.id/table]]@@ -70,7 +70,7 @@[:direction {:optional true} [:maybe [:ref ::lib.schema.order-by/direction]]]]])(mr/def ::test-literal-expression-spec- [:map+ [:map {:closed true}[:type [:= {:decode/normalize lib.schema.common/normalize-keyword} :literal]][:value [:ref ::literal/literal]]])@@ -79,7 +79,7 @@keyword?)(mr/def ::test-operator-expression-spec- [:map+ [:map {:closed true}[:type [:= {:decode/normalize lib.schema.common/normalize-keyword} :operator]][:operator ::test-operator-spec][:args {:default []} [:sequential [:ref ::test-expression-spec]]]])@@ -92,27 +92,27 @@[:operator [:ref ::test-operator-expression-spec]]])(mr/def ::test-segment-spec- [:map+ [:map {:closed true}[:type [:= {:decode/normalize lib.schema.common/normalize-keyword} :segment]][:id [:ref ::lib.schema.id/segment]]])(mr/def ::test-named-expression-spec- [:map+ [:map {:closed true}[:name string?][:value [:ref ::test-expression-spec]]])(mr/def ::test-measure-spec- [:map+ [:map {:closed true}[:type [:= {:decode/normalize lib.schema.common/normalize-keyword} :measure]][:id [:ref ::lib.schema.id/measure]]])(mr/def ::test-metric-spec- [:map+ [:map {:closed true}[:type [:= {:decode/normalize lib.schema.common/normalize-keyword} :metric]][:id [:ref ::lib.schema.id/metric]]])(mr/def ::test-join-spec- [:map+ [:map {:closed true}[:source [:ref ::test-source-spec]][:strategy ::lib.schema.join/strategy][:conditions {:optional true} [:maybe [:sequential ::test-join-condition-spec]]]])@@ -125,7 +125,7 @@[:operator [:ref ::test-operator-expression-spec]]])(mr/def ::test-join-condition-spec- [:map+ [:map {:closed true}[:operator {:decode/normalize lib.schema.common/normalize-keyword} ::test-operator-spec][:left [:ref ::test-join-source-spec]][:right [:ref ::test-join-source-spec]]])@@ -138,7 +138,7 @@[:ref ::test-metric-spec]])(mr/def ::test-stage-spec- [:map+ [:map {:closed true}[:source {:optional true} [:maybe ::test-source-spec]][:fields {:optional true} [:maybe [:sequential ::test-column-spec]]][:expressions {:optional true} [:maybe [:sequential ::test-named-expression-spec]]]@@ -150,7 +150,7 @@[:limit {:optional true} [:maybe number?]]])(mr/def ::test-query-spec- [:map+ [:map {:closed true}[:stages [:sequential ::test-stage-spec]]])(mr/def ::test-common-spec@@ -210,6 +210,6 @@::test-template-tag-spec])(mr/def ::test-native-query-spec- [:map+ [:map {:closed true}[:query string?][:template-tags {:optional true :default []} [:maybe [:ref ::test-template-tags-spec]]]])diff --git a/src/metabase/lib/schema/util.cljc b/src/metabase/lib/schema/util.cljcindex bc376806f67..40a9f3c4f60 100644--- a/src/metabase/lib/schema/util.cljc+++ b/src/metabase/lib/schema/util.cljc@@ -2,7 +2,6 @@(:refer-clojure :exclude [ref run! every? mapv reduce empty? first second])(:require[medley.core :as m]- [metabase.lib.options :as lib.options][metabase.util.malli :as mu][metabase.util.malli.registry :as mr][metabase.util.performance :as perf :refer [run! every? mapv reduce empty? first second]]))@@ -10,7 +9,7 @@(declare collect-uuids*)(defn- collect-uuids-in-map [m result]- (when-let [our-uuid (or (:lib/uuid (lib.options/options m))+ (when-let [our-uuid (or (:lib/uuid (:lib/options m))(:lib/uuid m))];; Keep duplicates in metadata of the result.(if (@result our-uuid)@@ -83,7 +82,7 @@(mu/defn mbql-clause-distinct-key"For deduplicating MBQL clauses: keep just the keys in options that are essential to distinguish one clause fromanother. Removes namespaced keywords and type information keys like `:base-type`."- [clause]+ [clause :- :metabase.lib.schema.common/any-clause](let [tag (first clause)opts (second clause)f #(cond-> %diff --git a/src/metabase/lib/segment.cljc b/src/metabase/lib/segment.cljcindex f0e7415d9c3..e1a902941c1 100644--- a/src/metabase/lib/segment.cljc+++ b/src/metabase/lib/segment.cljc@@ -70,7 +70,7 @@(mu/defn available-segments :- [:maybe [:sequential {:min 1} ::lib.schema.metadata/segment]]"Get a list of Segments that you may consider using as filter for a query. Only Segments that have the same`table-id` as the `source-table` for this query will be suggested."- ([query]+ ([query :- ::lib.schema/query](available-segments query -1))([query :- ::lib.schema/querystage-number :- :int]diff --git a/src/metabase/lib/stage.cljc b/src/metabase/lib/stage.cljcindex 05d15f46fb2..dffeb49ef91 100644--- a/src/metabase/lib/stage.cljc+++ b/src/metabase/lib/stage.cljc@@ -72,7 +72,7 @@(mu/defn- breakouts-columns :- [:maybe ::lib.metadata.calculation/visible-columns][query :- ::lib.schema/querystage-number :- :int- options :- [:maybe ::lib.metadata.calculation/returned-columns.options]]+ options :- [:maybe ::lib.metadata.calculation/visible-columns.options]](let [cols (lib.breakout/breakouts-metadata query stage-number)](not-empty(concat@@ -92,7 +92,7 @@(mu/defn- fields-columns :- [:maybe ::lib.metadata.calculation/visible-columns][query :- ::lib.schema/querystage-number :- :int- options :- [:maybe ::lib.metadata.calculation/returned-columns.options]]+ options :- [:maybe ::lib.metadata.calculation/visible-columns.options]](let [stage (lib.util/query-stage query stage-number);; this key is added by [[metabase.query-processor.middleware.add-implicit-clauses/add-implicit-fields]]; we;; forward it as `:qp/implicit-field?`@@ -112,7 +112,7 @@(mu/defn- summary-columns :- [:maybe ::lib.metadata.calculation/visible-columns][query :- ::lib.schema/querystage-number :- :int- options :- [:maybe ::lib.metadata.calculation/returned-columns.options]]+ options :- [:maybe ::lib.metadata.calculation/visible-columns.options]](not-empty(concat(breakouts-columns query stage-number options)@@ -124,7 +124,7 @@with [[lib.field.util/update-keys-for-col-from-previous-stage]]."[query :- ::lib.schema/querystage-number :- :int- options :- [:maybe ::lib.metadata.calculation/returned-columns.options]]+ options :- [:maybe ::lib.metadata.calculation/visible-columns.options]](when-let [previous-stage-number (lib.util/previous-stage-number query stage-number)](not-empty(into []@@ -139,7 +139,7 @@[query :- ::lib.schema/querystage-number :- :intcard-id :- [:maybe ::lib.schema.id/card]- options :- ::lib.metadata.calculation/returned-columns.options]+ options :- ::lib.metadata.calculation/visible-columns.options](when card-id(when-let [card (lib.metadata/card query card-id)](not-empty@@ -162,7 +162,7 @@(mu/defn- expressions-metadata :- [:maybe ::lib.metadata.calculation/visible-columns][query :- ::lib.schema/querystage-number :- :int- {:keys [include-late-exprs?]} :- [:map [:include-late-exprs? {:optional true} :boolean]]]+ {:keys [include-late-exprs?]} :- [:map {:closed true} [:include-late-exprs? {:optional true} :boolean]]](not-empty(for [[clause col] (map vector(:expressions (lib.util/query-stage query stage-number))@@ -298,7 +298,7 @@[query :- ::lib.schema/querystage-number :- :intstage :- ::lib.schema/stage- {:keys [include-remaps?], :as options} :- [:maybe ::lib.metadata.calculation/returned-columns.options]]+ {:keys [include-remaps?], :as options} :- [:maybe ::lib.metadata.calculation/visible-columns.options]];; Not including the stage itself in the cache key, since it's not used(!)(lib.computed/with-cache-ephemeral* query [::returned-columns stage-number (lib.metadata.calculation/cacheable-options options)](fn []diff --git a/src/metabase/lib/stage/util.cljc b/src/metabase/lib/stage/util.cljcindex a819bd4b789..8af57def6ac 100644--- a/src/metabase/lib/stage/util.cljc+++ b/src/metabase/lib/stage/util.cljc@@ -22,12 +22,12 @@(mu/defn append-stage :- ::lib.schema/query"Adds a new blank stage to the end of the pipeline."- [query]+ [query :- ::lib.schema/query](update query :stages conj {:lib/type :mbql.stage/mbql}))(mu/defn drop-stage :- ::lib.schema/query"Drops the final stage in the pipeline, will no-op if it is the only stage"- [query]+ [query :- ::lib.schema/query](if (= 1 (count (:stages query)))query(update query :stages pop)))diff --git a/src/metabase/lib/temporal_bucket.cljc b/src/metabase/lib/temporal_bucket.cljcindex 7ab3c54b352..3f20e677d65 100644--- a/src/metabase/lib/temporal_bucket.cljc+++ b/src/metabase/lib/temporal_bucket.cljc@@ -11,6 +11,8 @@[metabase.lib.metadata.calculation :as lib.metadata.calculation][metabase.lib.schema :as lib.schema][metabase.lib.schema.common :as lib.schema.common]+ [metabase.lib.schema.mbql-clause :as lib.schema.mbql-clause]+ [metabase.lib.schema.metadata :as lib.schema.metadata][metabase.lib.schema.temporal-bucketing :as lib.schema.temporal-bucketing][metabase.util :as u][metabase.util.i18n :as i18n]@@ -23,7 +25,7 @@([](describe-temporal-unit 1 nil))- ([unit]+ ([unit :- [:maybe :keyword]](describe-temporal-unit 1 unit))([n :- :int@@ -57,6 +59,14 @@(def ^:private TemporalIntervalAmount[:or [:enum :current :last :next] :int])+(def ^:private Bucketable+ "Something that can have a temporal bucket added to or read from it: an MBQL clause, a column, or a temporal+ bucketing option."+ [:or+ ::lib.schema.mbql-clause/clause+ ::lib.schema.metadata/column+ ::lib.schema.temporal-bucketing/option])+(defn- interval-n->int [n](if (number? n)n@@ -89,7 +99,7 @@(describe-temporal-interval n unit nil))([n :- TemporalIntervalAmountunit :- [:maybe :keyword]- opts :- [:maybe [:map [:include-current {:optional true} :boolean]]]]+ opts :- [:maybe [:map {:closed true} [:include-current {:optional true} :boolean]]]](let [n (interval-n->int n)unit (or unit :day)include-current? (:include-current opts)]@@ -305,9 +315,10 @@[:field 1 {:temporal-unit :day}]Pass a `nil` `unit` to remove the temporal bucket."- [x option-or-unit :- [:maybe [:or- ::lib.schema.temporal-bucketing/option- ::lib.schema.temporal-bucketing/unit]]]+ [x :- Bucketable+ option-or-unit :- [:maybe [:or+ ::lib.schema.temporal-bucketing/option+ ::lib.schema.temporal-bucketing/unit]]](with-temporal-bucket-method x (cond-> option-or-unit(not (keyword? option-or-unit)) :unit)))@@ -327,12 +338,12 @@(mu/defn raw-temporal-bucket :- [:maybe ::lib.schema.temporal-bucketing/unit]"Get the raw temporal bucketing `unit` associated with something e.g. a `:field` ref or a ColumnMetadata."- [x]+ [x :- Bucketable](temporal-bucket-method x))(mu/defn temporal-bucket :- [:maybe ::lib.schema.temporal-bucketing/option]"Get the current temporal bucketing option associated with something, if any."- [x]+ [x :- Bucketable](when-let [unit (raw-temporal-bucket x)]{:lib/type :option/temporal-bucketing:unit unit}))@@ -435,18 +446,19 @@(mu/defn available-temporal-buckets :- [:sequential [:ref ::lib.schema.temporal-bucketing/option]]"Get a set of available temporal bucketing units for `x`. Returns nil if no units are available."- ([query x]+ ([query :- ::lib.schema/query+ x :- Bucketable](available-temporal-buckets query -1 x))([query :- ::lib.schema/querystage-number :- :int- x]+ x :- Bucketable](available-temporal-buckets-method query stage-number x)))(mu/defn describe-temporal-pair :- :string"Return a string describing the temporal pair.Used when comparing temporal values like `[:!= ... [:field {:temporal-unit :day-of-week} ...] \"2022-01-01\"]`"- [temporal-column+ [temporal-column :- Bucketabletemporal-value :- [:or :int :string]](u.time/format-unit temporal-value (:unit (temporal-bucket temporal-column))))diff --git a/src/metabase/lib/underlying.cljc b/src/metabase/lib/underlying.cljcindex 4a714bd06e9..89b7ee99976 100644--- a/src/metabase/lib/underlying.cljc+++ b/src/metabase/lib/underlying.cljc@@ -12,6 +12,7 @@[metabase.lib.metadata.calculation :as lib.metadata.calculation][metabase.lib.ref :as lib.ref][metabase.lib.schema :as lib.schema]+ [metabase.lib.schema.drill-thru :as lib.schema.drill-thru][metabase.lib.schema.metadata :as lib.schema.metadata][metabase.lib.util :as lib.util][metabase.util.malli :as mu]@@ -74,7 +75,7 @@(top-level-column query column :rename-superfluous-options? true))([query :- ::lib.schema/querycolumn :- ::lib.schema.metadata/column- & {:keys [rename-superfluous-options?]} :- TopLevelColumnOptions]+ & {:keys [rename-superfluous-options?]} :- [:maybe TopLevelColumnOptions]](let [top-query (top-level-query query)](if (= query top-query)column ;; Unchanged if this is already a top-level query. That includes keeping the "superfluous" options!@@ -146,7 +147,7 @@See issue #66715."[query :- ::lib.schema/query- dimensions :- [:maybe [:sequential :map]]]+ dimensions :- [:maybe ::lib.schema.drill-thru/context.row]](not-empty(for [dim dimensions:when (top-level-column query (:column dim))]diff --git a/src/metabase/lib/util.cljc b/src/metabase/lib/util.cljcindex 5a68195023a..5b7372e0717 100644--- a/src/metabase/lib/util.cljc+++ b/src/metabase/lib/util.cljc@@ -16,6 +16,9 @@[metabase.lib.schema :as lib.schema][metabase.lib.schema.expression :as lib.schema.expression][metabase.lib.schema.id :as lib.schema.id]+ [metabase.lib.schema.join :as lib.schema.join]+ [metabase.lib.schema.mbql-clause :as lib.schema.mbql-clause]+ [metabase.lib.schema.metadata :as lib.schema.metadata][metabase.lib.schema.ref :as lib.schema.ref][metabase.util :as u][metabase.util.i18n :as i18n]@@ -294,13 +297,13 @@(dissoc query :type :query)))(mr/def ::legacy-query- [:map- {:error/message "legacy query"}+ "A legacy MBQL query: [[pipeline]] only reads its `:type`, and its inner query may not be normalized yet."+ [:map {:closed false, ::mr/deliberately-open true, :error/message "legacy query"}[:type [:enum :native :query]]])(mr/def ::mbql5-query- [:map- {:error/message "MBQL 5 query"}+ "An MBQL 5 query that may still be being built, e.g. by [[metabase.lib.convert]] before its clauses get `:lib/uuid`s."+ [:map {:closed false, ::mr/deliberately-open true, :error/message "MBQL 5 query"}[:lib/type [:= :mbql/query]]])(mr/def ::legacy-or-mbql5-query@@ -322,11 +325,12 @@(mu/defn canonical-stage-index :- [:int {:min 0}]"If `stage-number` index is a negative number e.g. `-1` convert it to a positive index so we can use `nth` on`stages`. `-1` = the last stage, `-2` = the penultimate stage, etc."- [{:keys [stages], :as _query} :- :map- stage-number :- :int]- (let [stage-number' (if (neg? stage-number)- (+ (count stages) stage-number)- stage-number)]+ [query :- ::legacy-or-mbql5-query+ stage-number :- :int]+ (let [{:keys [stages]} (pipeline query)+ stage-number' (if (neg? stage-number)+ (+ (count stages) stage-number)+ stage-number)](when (or (>= stage-number' (count stages))(neg? stage-number'))(throw (ex-info (i18n/tru "Stage {0} does not exist" stage-number)@@ -335,7 +339,7 @@(mu/defn previous-stage-number :- [:maybe [:int {:min 0}]]"The index of the previous stage, if there is one. `nil` if there is no previous stage."- [query :- :map+ [query :- ::legacy-or-mbql5-querystage-number :- :int](let [stage-number (canonical-stage-index query stage-number)](when (pos? stage-number)@@ -348,11 +352,12 @@(mu/defn next-stage-number :- [:maybe :int]"The index of the next stage, if there is one. `nil` if there is no next stage."- [{:keys [stages], :as _query} :- :map- stage-number :- :int]- (let [stage-number (if (neg? stage-number)- (+ (count stages) stage-number)- stage-number)]+ [query :- ::legacy-or-mbql5-query+ stage-number :- :int]+ (let [{:keys [stages]} (pipeline query)+ stage-number (if (neg? stage-number)+ (+ (count stages) stage-number)+ stage-number)](when (< (inc stage-number) (count stages))(inc stage-number))))@@ -365,14 +370,15 @@(mu/defn query-stage :- [:maybe ::lib.schema/stage]"Fetch a specific `stage` of a query. This handles negative indices as well, e.g. `-1` will return the last stage ofthe query."- [query :- ::mbql5-query+ [query :- ::legacy-or-mbql5-querystage-number :- :int]- (let [{:keys [stages], :as query} query]+ (let [{:keys [stages], :as query} (pipeline query)](get (vec stages) (canonical-stage-index query stage-number))))(mu/defn previous-stage :- [:maybe ::lib.schema/stage]"Return the previous stage of the query, if there is one; otherwise return `nil`."- [query stage-number :- :int]+ [query :- ::legacy-or-mbql5-query+ stage-number :- :int](when-let [stage-num (previous-stage-number query stage-number)](query-stage query stage-num)))@@ -384,7 +390,16 @@`stage-number` can be a negative index, e.g. `-1` will update the last stage of the query."[query :- ::legacy-or-mbql5-querystage-number :- :int- f & args]+ f :- ifn?+ & args :- [:* [:maybe [:or+ :keyword+ ifn?+ :int+ :string+ ::lib.schema.mbql-clause/clause+ ::lib.schema.metadata/stage+ ::lib.schema/pivot+ [:sequential [:or :keyword :int ::lib.schema.mbql-clause/clause ::lib.schema.join/join ::lib.schema.metadata/column]]]]]](let [{:keys [stages], :as query} (pipeline query)stage-number' (canonical-stage-index query stage-number)stages' (apply update (vec stages) stage-number' f args)]@@ -414,7 +429,7 @@(mu/defn ensure-mbql-final-stage :- ::lib.schema/query"Convert query to a MBQL 5 (pipeline) query, and make sure the final stage is an `:mbql` one."- [query]+ [query :- ::legacy-or-mbql5-query](let [query (pipeline query)](cond-> query(native-stage? query -1)@@ -424,19 +439,19 @@"If `table-id` is a legacy `card__<id>`-style string, parse the `<id>` part to an integer Card ID. Only for legacyqueries! You don't need to use this in MBQL 5 since this is converted automatically by [[metabase.lib.convert]] to`:source-card`."- [table-id]+ [table-id :- [:or ::lib.schema.id/table :string]](when (string? table-id)(when-let [[_match card-id-str] (re-find #"^card__(\d+)$" table-id)](parse-long card-id-str))))(mu/defn source-table-id :- [:maybe ::lib.schema.id/table]"If this query has a `:source-table` ID, return it."- [query]+ [query :- ::lib.schema/query](-> query :stages first :source-table))(mu/defn source-card-id :- [:maybe ::lib.schema.id/card]"If this query has a `:source-card` ID, return it."- [query]+ [query :- ::lib.schema/query](-> query :stages first :source-card))(def ^:private strip-id-regex@@ -458,7 +473,7 @@[query :- ::lib.schema/querystage-number :- :intlocation :- [:enum :breakout :aggregation]- a-summary-clause]+ a-summary-clause :- ::lib.common/op-arg](let [query (pipeline query)stage-number (or stage-number -1)stage (query-stage query stage-number)@@ -518,19 +533,23 @@(throw (ex-info "Could not convert old :aggregation ref to new UUIDs"{:aggregation &match})))])))+(mr/def ::query-like+ "A query, stage, or metadata map in any state of normalization, string keys included, whose type+ [[normalized-query-type]] reads."+ [:map {:closed false, ::mr/deliberately-open true, :description "query-like map"}])+(mu/defn normalized-query-type :- [:maybe [:enum #_MBQL5 :mbql/query #_legacy :query :native #_audit :internal]]"Get the `:lib/type` or `:type` from `query`, even if it is not-yet normalized."- [query :- [:maybe :map]]- (when (map? query)- (when-let [query-type (some-> (some #(get query %)- [:lib/type :type "lib/type" "type"])- keyword)]- (when (#{:mbql/query :query :native :internal} query-type)- query-type))))+ [query :- [:maybe ::query-like]]+ (when-let [query-type (some-> (some #(get query %)+ [:lib/type :type "lib/type" "type"])+ keyword)]+ (when (#{:mbql/query :query :native :internal} query-type)+ query-type)))(mu/defn normalized-mbql-version :- [:maybe [:enum :mbql-version/mbql5 :mbql-version/legacy]]"Version of MBQL a `query` map is using, either `:mbql-version/mbql-5` or `:mbql-version/legacy`."- [query :- [:maybe :map]]+ [query :- [:maybe ::query-like]](case (normalized-query-type query):mbql/query :mbql-version/mbql5(:query :native) :mbql-version/legacydiff --git a/src/metabase/lib/util/unique_name_generator.cljc b/src/metabase/lib/util/unique_name_generator.cljcindex 448f8d907ae..f3378be4e99 100644--- a/src/metabase/lib/util/unique_name_generator.cljc+++ b/src/metabase/lib/util/unique_name_generator.cljc@@ -39,7 +39,7 @@(truncate-alias \"some_really_long_string\" 15) ; -> \"some_r_8e0f9bc2\"(truncate-alias \"some_really_long_string_2\" 15) ; -> \"some_r_2a3c73eb\""- ([s]+ ([s :- :string](truncate-alias s truncate-alias-max-length-bytes))([s :- :string@@ -70,7 +70,7 @@:string];; (f id str) => unique-str[:=>- [:cat :any :string]+ [:cat symbol? :string]:string]])(mu/defn- untruncated-unique-alias :- :stringdiff --git a/src/metabase/lib/value.cljc b/src/metabase/lib/value.cljcindex 0e5a11b4624..b042c9b805f 100644--- a/src/metabase/lib/value.cljc+++ b/src/metabase/lib/value.cljc@@ -14,5 +14,5 @@;; `:lib/uuid` is optional, as it will be added automatically[:map[:lib/uuid {:optional true} ::lib.schema.common/uuid]]]]- v]+ v :- ::lib.schema.literal/value.value][:value (assoc opts :lib/uuid (str (random-uuid))) v])diff --git a/src/metabase/lib/walk.cljc b/src/metabase/lib/walk.cljcindex 5c2b8def37f..20b00ebc82a 100644--- a/src/metabase/lib/walk.cljc+++ b/src/metabase/lib/walk.cljc@@ -7,6 +7,7 @@[metabase.lib.hierarchy :as lib.hierarchy][metabase.lib.join :as lib.join][metabase.lib.schema :as lib.schema]+ [metabase.lib.schema.expression :as lib.schema.expression][metabase.lib.schema.join :as lib.schema.join][metabase.lib.schema.mbql-clause :as lib.schema.mbql-clause][metabase.lib.util :as lib.util]@@ -151,7 +152,11 @@(fn [_query _path-type _path stage-or-join](when (:source-card stage-or-join)(reduced true))))))"- ([query f] (walk query f nil))+ ([query :- ::lib.schema/query+ f :- [:=>+ [:cat :map ::path-type ::path ::stage-or-join]+ ::walk-fn-result]]+ (walk query f nil))([query :- ::lib.schema/queryf :- [:=>[:cat :map ::path-type ::path ::stage-or-join]@@ -177,7 +182,9 @@"Like [[walk]], but only walks the stages in a query. `f` is invoked like(f query path stage) => updated-stage-or-nil"- ([query f] (walk-stages query f nil))+ ([query :- ::lib.schema/query+ f :- [:=> [:cat :map ::path ::lib.schema/stage] ::walk-stages-fn-result]]+ (walk-stages query f nil))([query :- ::lib.schema/queryf :- [:=> [:cat :map ::path ::lib.schema/stage] ::walk-stages-fn-result]opts :- [:maybe ::walk-opts]]@@ -235,6 +242,13 @@query' (assoc query :stages (:stages join))](recur query' (if (empty? more) [:stages 0] more))))))+(mr/def ::walked-clause+ "An MBQL clause, or a literal or keyword (e.g. a temporal unit) argument of one, as [[walk-clause]] visits it."+ [:or+ [:ref ::lib.schema.mbql-clause/clause]+ ::lib.schema.expression/expression+ :keyword])+(mu/defn apply-f-for-stage-at-path"Use a function that takes top-level `query` and `stage-number` with a `query` and `path`,via [[query-for-stage-at-path]]. Lets you use stuff like [[metabase.lib.aggregation/resolve-aggregation]] in@@ -248,7 +262,7 @@[f :- fn?query :- ::lib.schema/querystage-path :- ::path- & args]+ & args :- [:* [:or ::walked-clause [:ref ::lib.schema.join/join] [:ref :metabase.lib.metadata.calculation/visible-columns.options]]]](let [{:keys [query stage-number]} (query-for-path query stage-path)](apply f query stage-number args)))@@ -274,8 +288,7 @@(mu/defn join-last-stage-path :- ::path"Given a `join-path`, return the path to the last stage within the join."[join-path :- ::path- join :- [:map- [:lib/type [:= :mbql/join]]]]+ join :- [:ref ::lib.schema.join/join]](into (vec join-path) [:stages (dec (count (:stages join)))]))(declare walk-clause* walk-clauses*)@@ -357,8 +370,8 @@on every MBQL subclause of `clause`, then on `clause` itself. (Also includes non-clause arguments like the `1` and`2` in `[:= {} [:field {} 1] 2]`.)"- [clause :- :any- f :- [:=> [:cat :any] :any]]+ [clause :- ::walked-clause+ f :- [:=> [:cat ::walked-clause] [:maybe ::walked-clause]]](walk-clause* clause (walk-clause-wrap-f f)))(mu/defn walk-clauses*@@ -369,8 +382,8 @@(f clause)on every clause in the normal places clauses live in a query."- [clauses :- [:maybe [:sequential :any]]- f :- [:=> [:cat :any] :any]]+ [clauses :- [:maybe [:sequential ::walked-clause]]+ f :- [:=> [:cat ::walked-clause] [:maybe ::walked-clause]]];; we're doing this the hard way instead of using `mapv` to avoid allocating new objects/creating a new query map;; if we don't actually change anything(when clauses@@ -391,7 +404,7 @@for every clause in the stage."[stage :- ::lib.schema/stage- f :- [:=> [:cat :any] :any]]+ f :- [:=> [:cat ::walked-clause] [:maybe ::walked-clause]]](when (lib.util/mbql-stage? stage)(reduce(fn [stage k]@@ -418,7 +431,7 @@Avoids creating new objects except for when `f` actually returns something different."[query :- ::lib.schema/queryf :- [:=>- [:cat :map ::path-type ::path ::lib.schema.mbql-clause/clause]+ [:cat ::lib.schema/query ::path-type ::path ::lib.schema.mbql-clause/clause][:maybe ::lib.schema.mbql-clause/clause]]](walkquerydiff --git a/src/metabase/lib/walk/util.cljc b/src/metabase/lib/walk/util.cljcindex 6ef81e66c7d..24ab5aeef7f 100644--- a/src/metabase/lib/walk/util.cljc+++ b/src/metabase/lib/walk/util.cljc@@ -85,7 +85,7 @@(mu/defn all-segment-ids :- [:maybe [:set {:min 1} ::lib.schema.id/segment]]"Return a set of all segment IDs anywhere in the query."- [query]+ [query :- ::lib.schema/query](let [segment-ids (volatile! (transient #{}))](lib.walk/walk-clausesquery@@ -100,7 +100,7 @@(mu/defn all-measure-ids :- [:maybe [:set {:min 1} ::lib.schema.id/measure]]"Return a set of all measure IDs anywhere in the query."- [query]+ [query :- ::lib.schema/query](let [measure-ids (volatile! (transient #{}))](lib.walk/walk-clausesquery@@ -272,7 +272,7 @@(all-template-tags query))))(mr/def ::referenced-entity-ids- [:map+ [:map {:closed true}[:table [:set ::lib.schema.id/table]][:card [:set ::lib.schema.id/card]][:metric [:set ::lib.schema.id/metric]]@@ -281,7 +281,7 @@[:snippet [:set ::lib.schema.id/snippet]]])(mr/def ::referenced-entity-ids.options- [:map+ [:map {:closed true}[:include-implicitly-joinable? {:optional true} :boolean]])(mu/defn- implicitly-joinable-table-ids :- [:set ::lib.schema.id/table]diff --git a/src/metabase/lib_be/hash.clj b/src/metabase/lib_be/hash.cljindex d3f6e085802..56726254ddf 100644--- a/src/metabase/lib_be/hash.clj+++ b/src/metabase/lib_be/hash.clj@@ -35,7 +35,7 @@(mu/defn query->hash-input :- :map"Normalize and strip `query` to the canonical form used for hashing."- [query :- :map]+ [query :- :metabase.lib.util/query-like](-> query(cond-> (not= (keyword (:type query)) :internal)(as-> $query (lib-be.models.transforms/normalize-query nil $query {:strict? true})))@@ -43,7 +43,7 @@(mu/defn query-hash :- bytes?"Return a 256-bit SHA3 hash of `query` as a key for the cache. (This is returned as a byte array.)"- ^bytes [query :- :map]+ ^bytes [query :- :metabase.lib.util/query-like](-> queryquery->hash-inputjson/encodediff --git a/src/metabase/lib_be/metadata/bootstrap.clj b/src/metabase/lib_be/metadata/bootstrap.cljindex 129964590c7..e46c37ccf99 100644--- a/src/metabase/lib_be/metadata/bootstrap.clj+++ b/src/metabase/lib_be/metadata/bootstrap.clj@@ -11,12 +11,15 @@[metabase.util.malli.registry :as mr][metabase.util.performance :as perf]))+(def ^:private Query+ :metabase.lib.util/query-like)+(mu/defn- source-card-id-for-mbql5-query :- [:maybe ::lib.schema.id/card]- [query :- :map]+ [query :- Query](-> query :stages first :source-card))(mu/defn- source-card-id-for-legacy-query :- [:maybe ::lib.schema.id/card]- [query :- :map]+ [query :- Query](let [inner-query (:query query)deepest-inner-query (loop [inner-query inner-query](let [source-query (:source-query inner-query)]@@ -68,7 +71,7 @@(:database-id card)))(mu/defn- source-card-id :- ::lib.schema.id/card- [query :- :map]+ [query :- Query](case (lib/normalized-query-type query):mbql/query(source-card-id-for-mbql5-query query)@@ -82,7 +85,7 @@(mu/defn- resolved-database-id :- [:maybe ::lib.schema.id/database][metadata-provider :- [:maybe ::lib.metadata.protocols/metadata-provider]- query :- :map]+ query :- Query](let [database-id (:database query)](cond(pos-int? database-id)@@ -110,15 +113,11 @@[:database ::lib.schema.id/database]]]]"If query has `:database` `-1337` (the legacy database ID for queries using a source Card that had an unknowndatabase), resolve the correct database ID and assoc it into the query."- ([query]+ ([query :- [:maybe [:or ::empty-map Query]]](resolve-database nil query))([metadata-provider :- [:maybe ::lib.metadata.protocols/metadata-provider]- query :- [:maybe- [:or- ::empty-map- [:map- [:database ::maybe-unresolved-database-id]]]]]+ query :- [:maybe [:or ::empty-map Query]]](when (seq query)(if (pos-int? (:database query))querydiff --git a/src/metabase/lib_be/metadata/jvm.clj b/src/metabase/lib_be/metadata/jvm.cljindex b0fedaf5f88..1532d00e0af 100644--- a/src/metabase/lib_be/metadata/jvm.clj+++ b/src/metabase/lib_be/metadata/jvm.clj@@ -6,6 +6,7 @@[clojure.core.cache :as cache][clojure.core.cache.wrapped :as cache.wrapped][clojure.string :as str]+ [malli.core :as mc][metabase.lib-be.db :as lib-be.db][metabase.lib.metadata.cached-provider :as lib.metadata.cached-provider][metabase.lib.metadata.invocation-tracker :as lib.metadata.invocation-tracker]@@ -19,6 +20,7 @@[metabase.util :as u][metabase.util.json :as json][metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr][metabase.util.memoize :as u.memo][metabase.util.performance :as perf :refer [get-in]][metabase.util.snake-hating-map :as u.snake-hating-map]@@ -46,10 +48,89 @@get a nice performance boost."(u.memo/fast-memo u/->kebab-case-en))+(mr/def ::metadata-column-row+ "A Field row as the `:metadata/column` select returns it, with the columns of its Dimension and FieldValues."+ [:merge+ :metabase.warehouse-schema.schema/field+ [:map {:closed true}+ [:dimension/human_readable_field_id [:maybe ::lib.schema.id/field]]+ [:dimension/id [:maybe pos-int?]]+ [:dimension/name [:maybe :string]]+ [:dimension/type [:maybe :string]]+ [:values/human_readable_values [:maybe :string]]+ [:values/values [:maybe :string]]]])++(mr/def ::model-field+ "A `:model/Field` instance: a Field row, or a Card result metadata column X-Rays tags as a Field."+ [:or+ :metabase.warehouse-schema.schema/field+ :metabase.legacy-mbql.schema/legacy-column-metadata])++(mr/def ::instance+ "A Toucan 2 instance [[instance->metadata]] converts, by its model, or a legacy result metadata column."+ [:multi {:dispatch (fn [instance] (or (t2/model instance)+ (when (= (:lib/type instance) :metadata/column) ::lib-column)+ ::legacy-column))+ :lazy-refs true}+ [::legacy-column :metabase.legacy-mbql.schema/legacy-column-metadata]+ [::lib-column ::lib.schema.metadata/column]+ [:metadata/database :metabase.warehouses.schema/database]+ [:metadata/table :metabase.warehouse-schema.schema/table]+ [:metadata/native-query-snippet :metabase.native-query-snippets.schema/native-query-snippet]+ [:metadata/transform :metabase.transforms.schema/transform]+ [:metadata/column ::metadata-column-row]+ [:metadata/card :metabase.queries.schema/card]+ [:metadata/metric :metabase.queries.schema/card]+ [:metadata/segment :metabase.segments.schema/segment]+ [:metadata/measure :metabase.measures.schema/measure]+ [:model/Database :metabase.warehouses.schema/database]+ [:model/Table :metabase.warehouse-schema.schema/table]+ [:model/Field ::model-field]+ [:model/Card :metabase.queries.schema/card]+ [:model/Segment :metabase.segments.schema/segment]+ [:model/Measure :metabase.measures.schema/measure]+ [:model/NativeQuerySnippet :metabase.native-query-snippets.schema/native-query-snippet]+ [:model/Transform :metabase.transforms.schema/transform]])+(def ^:private metadata-type->schema{:metadata/card ::lib.schema.metadata/card:metadata/column ::lib.schema.metadata/column})+(def ^:private metadata-type->lib-schema+ {:metadata/card ::lib.schema.metadata/card+ :metadata/database ::lib.schema.metadata/database+ :metadata/measure ::lib.schema.metadata/measure+ :metadata/metric ::lib.schema.metadata/metric+ :metadata/native-query-snippet ::lib.schema.metadata/native-query-snippet+ :metadata/segment ::lib.schema.metadata/segment+ :metadata/table ::lib.schema.metadata/table+ :metadata/transform ::lib.schema.metadata/transform})++(defn- schema-keys+ [schema]+ (let [schema (mc/deref-all (mr/resolve-schema schema))]+ (case (mc/type schema)+ :map (into #{} (map first) (mc/children schema))+ :and (perf/some schema-keys (mc/children schema))+ nil)))++(def ^:private metadata-type->keys+ "The keys the Lib metadata schema of a metadata type declares, by metadata type."+ (u.memo/fast-memo (fn [metadata-type]+ (some-> (metadata-type->lib-schema metadata-type) schema-keys))))++(defn- drop-undeclared-columns+ "`instance` without the unqualified keys the Lib metadata schema of `metadata-type` doesn't declare."+ [instance metadata-type]+ (if-let [declared (metadata-type->keys metadata-type)]+ (reduce-kv (fn [m k _v]+ (if (or (qualified-keyword? k) (contains? declared k))+ m+ (dissoc m k)))+ instance+ instance)+ instance))+;; TODO (Cam 2026-08-27) Consider whether we should just have this be the normal behavior for normalizing;; application-database-style metadata to Lib-style metadata, e.g. why can't we just use;;@@ -59,7 +140,7 @@(mu/defn instance->metadata"Convert a (presumably) Toucan 2 instance of an application database model with `snake_case` keys to a Lib stylemetadata instance with `:lib/type` and `kebab-case` keys."- [instance :- :map+ [instance :- ::instancemetadata-type :- :keyword](let [normalize (if-let [schema (get metadata-type->schema metadata-type)](fn [instance]@@ -68,6 +149,7 @@(-> instance(perf/update-keys memoized-kebab-key)(assoc :lib/type metadata-type)+ (drop-undeclared-columns metadata-type)normalizeu.snake-hating-map/snake-hating-map(vary-meta assoc :metabase/toucan-instance instance))))diff --git a/src/metabase/lib_be/models/transforms.clj b/src/metabase/lib_be/models/transforms.cljindex 91ff9de7c71..21202fe942d 100644--- a/src/metabase/lib_be/models/transforms.clj+++ b/src/metabase/lib_be/models/transforms.clj@@ -18,7 +18,7 @@(defn- normalize-mbql[query]- (case (lib/normalized-query-type query)+ (case (when (map? query) (lib/normalized-query-type query)):mbql/query (lib.schema.common/normalize-map-no-kebab-case query)(:query :native) (mbql.normalize/normalize query)(lib.schema.common/normalize-map-no-kebab-case query)))@@ -28,14 +28,15 @@[true [:= {:description "empty map"} {}]][false ::lib.schema/query]]]"Normalize an MBQL `query` to MBQL 5 and attach a metadata provider."- ([query]+ ([query :- [:maybe :metabase.lib.util/query-like]](normalize-query nil query))- ([metadata-providerable query]+ ([metadata-providerable :- [:maybe ::lib.metadata.protocols/metadata-providerable]+ query :- [:maybe :metabase.lib.util/query-like]](normalize-query metadata-providerable query nil))([metadata-providerable :- [:maybe ::lib.metadata.protocols/metadata-providerable]- query :- [:maybe :map]+ query :- [:maybe :metabase.lib.util/query-like]{:keys [strict?]} :- [:maybe[:map{:closed true}diff --git a/src/metabase/lib_be/schema.clj b/src/metabase/lib_be/schema.cljindex 41526556a00..ed4a9990d19 100644--- a/src/metabase/lib_be/schema.clj+++ b/src/metabase/lib_be/schema.clj@@ -4,10 +4,11 @@[metabase.lib-be.models.transforms :as lib-be.transforms][metabase.lib.core :as lib][metabase.lib.schema :as lib.schema]+ [metabase.lib.schema.info :as lib.schema.info][metabase.lib.schema.middleware-options :as lib.schema.middleware-options]- [metabase.lib.util :as lib.util][metabase.util.i18n :refer [deferred-tru tru]][metabase.util.malli.registry :as mr]+ [metabase.util.malli.schema :as ms][metabase.util.performance :refer [empty?]]))(set! *warn-on-reflection* true)@@ -32,17 +33,36 @@:decode/normalize #(normalize-query % (tru "value must be a valid MBQL query."))}[:ref ::lib.schema/query]])+(mr/def ::internal-query.scope+ "Mirrors `metabase.actions.types/scope.normalized`. Duplicated (rather than referenced) so this namespace does not+ have to depend on the `actions` module."+ [:or+ [:map {:closed true} [:type :keyword] [:dashboard-id ms/PositiveInt]]+ [:map {:closed true} [:type :keyword] [:dashcard-id ms/PositiveInt]]+ [:map {:closed true} [:type :keyword] [:card-id ms/PositiveInt]]+ [:map {:closed true} [:type :keyword] [:model-id ms/PositiveInt]]+ [:map {:closed true} [:type :keyword] [:table-id ms/PositiveInt]]+ [:map {:closed true} [:type :keyword] [:webhook-id ms/PositiveInt]]+ [:map {:closed true} [:type :keyword] [:unknown [:enum :model-action]]]])+(mr/def ::internal-query+ "An internal (audit) query: either the QP's own `:fn`/`:args` invocation shape, or the action-descriptor shape+ stashed on a [[metabase.actions.audit/Base]] row's `:template` for a `:model.row/*` or similar action."[:map {:closed true}- [:type [:= {:decode/normalize keyword} :internal]]- [:fn :string]- [:args {:optional true} [:maybe [:sequential [:maybe [:or :string number? :boolean]]]]]- [:limit {:optional true} [:maybe :int]]- [:offset {:optional true} [:maybe :int]]- [:middleware {:optional true} [:ref ::lib.schema.middleware-options/middleware-options]]])+ [:type [:= {:decode/normalize keyword} :internal]]+ [:fn {:optional true} :string]+ [:args {:optional true} [:maybe [:sequential [:maybe [:or :string number? :boolean]]]]]+ [:limit {:optional true} [:maybe :int]]+ [:offset {:optional true} [:maybe :int]]+ [:middleware {:optional true} [:ref ::lib.schema.middleware-options/middleware-options]]+ [:info {:optional true} [:maybe [:ref ::lib.schema.info/info]]]+ [:action {:optional true} [:or :string :keyword]]+ [:database {:optional true} [:maybe pos-int?]]+ [:scope {:optional true} ::internal-query.scope]+ [:action-id {:optional true} [:maybe pos-int?]]])(mr/def ::maybe-legacy-or-internal-query- [:multi {:dispatch (fn [query] (and (map? query) (= (lib.util/normalized-query-type query) :internal)))+ [:multi {:dispatch (fn [query] (and (map? query) (contains? #{:internal "internal"} (or (:type query) (get query "type"))))):description (deferred-tru "value must be a valid MBQL query, or an internal audit query.")}[true ::internal-query][false ::maybe-legacy-query]])diff --git a/src/metabase/lib_metric/ast/compile.cljc b/src/metabase/lib_metric/ast/compile.cljcindex e8a7705132c..02611f66bb2 100644--- a/src/metabase/lib_metric/ast/compile.cljc+++ b/src/metabase/lib_metric/ast/compile.cljc@@ -12,6 +12,11 @@(defn- random-uuid-str [](str (random-uuid)))+(def ^:private Options+ "Options accepted by [[compile-to-mbql]] and [[compile-to-values-query]]."+ [:map {:closed true}+ [:limit {:optional true} [:maybe pos-int?]]])+;;; -------------------- Resolution --------------------(defn- find-mapping@@ -272,7 +277,7 @@Options:- :limit - add limit to query"[{:keys [source] :as source-query} :- ::ast.schema/source-query- & {:as opts}]+ & {:as opts} :- [:maybe Options]](if (needs-two-stage? source)(compile-two-stage-query source-query opts)(compile-single-stage-query source-query opts)))@@ -336,7 +341,7 @@Options:- :limit - add limit to query"[{:keys [source] :as source-query} :- ::ast.schema/source-query- & {:as opts}]+ & {:as opts} :- [:maybe Options]](if (needs-two-stage? source)(compile-two-stage-values-query source-query opts)(compile-single-stage-values-query source-query opts)))diff --git a/src/metabase/lib_metric/ast/schema.cljc b/src/metabase/lib_metric/ast/schema.cljcindex 6ab9e75f1db..8bcdd4b8d33 100644--- a/src/metabase/lib_metric/ast/schema.cljc+++ b/src/metabase/lib_metric/ast/schema.cljc@@ -5,31 +5,36 @@(:require[metabase.lib-metric.operators :as operators][metabase.lib-metric.schema :as lib-metric.schema]+ [metabase.lib.schema.join :as lib.schema.join]+ [metabase.lib.schema.literal :as lib.schema.literal]+ [metabase.lib.schema.mbql-clause :as lib.schema.mbql-clause]+ [metabase.lib.schema.metadata :as lib.schema.metadata][metabase.util.malli.registry :as mr]));;; -------------------- Primitive Nodes --------------------(mr/def ::table-node"Reference to a database table."- [:map+ [:map {:closed true}[:node/type [:= :ast/table]][:id pos-int?][:name {:optional true} [:maybe string?]]])(mr/def ::column-node"Reference to a database column/field."- [:map+ [:map {:closed true}[:node/type [:= :ast/column]][:id [:or pos-int? string?]][:name {:optional true} [:maybe string?]][:table-id {:optional true} [:maybe pos-int?]]+ [:source-field {:optional true} [:maybe pos-int?]][:base-type {:optional true} [:maybe keyword?]]]);;; -------------------- Dimension Nodes --------------------(mr/def ::dimension-node"A dimension definition - the abstract dimension."- [:map+ [:map {:closed true}[:node/type [:= :ast/dimension]][:id ::lib-metric.schema/dimension-id][:name {:optional true} [:maybe string?]]@@ -40,24 +45,28 @@(mr/def ::dimension-ref-options"Options for dimension references (bucketing, binning, etc.)."- [:map+ [:map {:closed true}+ [:lib/uuid {:optional true} [:maybe :string]]+ [:display-name {:optional true} [:maybe :string]]+ [:effective-type {:optional true} [:maybe keyword?]]+ [:semantic-type {:optional true} [:maybe keyword?]][:temporal-unit {:optional true} [:maybe keyword?]]- [:binning {:optional true} [:maybe :map]]])+ [:binning {:optional true} [:maybe ::lib-metric.schema/binning]]])(mr/def ::dimension-ref-node"A reference to a dimension, used in filters and group-by."- [:map+ [:map {:closed true}[:node/type [:= :ast/dimension-ref]][:dimension-id ::lib-metric.schema/dimension-id][:options {:optional true} [:maybe ::dimension-ref-options]]])(mr/def ::dimension-expression-node"A dimension reference wrapped in an expression (e.g. temporal extraction like :get-day-of-week)."- [:map+ [:map {:closed true}[:node/type [:= :ast/dimension-expression]][:expression-op keyword?][:dimension ::dimension-ref-node]- [:args {:optional true} [:maybe [:sequential :any]]]])+ [:args {:optional true} [:maybe [:sequential [:or :keyword ::lib.schema.literal/literal]]]]])(mr/def ::dimension-or-expression"A dimension reference or an expression wrapping one."@@ -65,7 +74,7 @@(mr/def ::dimension-mapping-node"Connects a dimension to a physical column."- [:map+ [:map {:closed true}[:node/type [:= :ast/dimension-mapping]][:dimension-id ::lib-metric.schema/dimension-id][:table-id {:optional true} [:maybe pos-int?]]@@ -74,40 +83,40 @@;;; -------------------- Aggregation Nodes --------------------(mr/def ::aggregation-count- [:map+ [:map {:closed true}[:node/type [:= :aggregation/count]][:column {:optional true} [:maybe ::column-node]]])(mr/def ::aggregation-sum- [:map+ [:map {:closed true}[:node/type [:= :aggregation/sum]][:column ::column-node]])(mr/def ::aggregation-avg- [:map+ [:map {:closed true}[:node/type [:= :aggregation/avg]][:column ::column-node]])(mr/def ::aggregation-min- [:map+ [:map {:closed true}[:node/type [:= :aggregation/min]][:column ::column-node]])(mr/def ::aggregation-max- [:map+ [:map {:closed true}[:node/type [:= :aggregation/max]][:column ::column-node]])(mr/def ::aggregation-distinct- [:map+ [:map {:closed true}[:node/type [:= :aggregation/distinct]][:column ::column-node]])(mr/def ::aggregation-mbql"For complex/custom aggregations that don't fit standard types."- [:map+ [:map {:closed true}[:node/type [:= :aggregation/mbql]]- [:clause :any]])+ [:clause ::lib.schema.mbql-clause/clause]])(mr/def ::aggregation-node"Union of all aggregation node types."@@ -124,58 +133,58 @@(mr/def ::filter-comparison"Comparison filter (=, !=, <, <=, >, >=)."- [:map+ [:map {:closed true}[:node/type [:= :filter/comparison]][:operator [:enum := :!= :< :<= :> :>=]][:dimension ::dimension-or-expression]- [:values [:sequential :any]]])+ [:values [:sequential ::lib.schema.literal/literal]]])(mr/def ::filter-between"Between filter for range checks."- [:map+ [:map {:closed true}[:node/type [:= :filter/between]][:dimension ::dimension-or-expression]- [:min :any]- [:max :any]])+ [:min ::lib.schema.literal/literal]+ [:max ::lib.schema.literal/literal]])(mr/def ::filter-string"String filter operations."- [:map+ [:map {:closed true}[:node/type [:= :filter/string]][:operator [:enum :contains :starts-with :ends-with :does-not-contain]][:dimension ::dimension-or-expression][:value string?]- [:options {:optional true} [:map [:case-sensitive {:optional true} [:maybe boolean?]]]]])+ [:options {:optional true} [:map {:closed true} [:case-sensitive {:optional true} [:maybe boolean?]]]]])(mr/def ::filter-null"Null/empty check filter."- [:map+ [:map {:closed true}[:node/type [:= :filter/null]][:operator [:enum :is-null :not-null :is-empty :not-empty]][:dimension ::dimension-or-expression]])(mr/def ::filter-in"Multi-value filter (in, not-in)."- [:map+ [:map {:closed true}[:node/type [:= :filter/in]][:operator [:enum :in :not-in]][:dimension ::dimension-or-expression]- [:values [:sequential :any]]])+ [:values [:sequential ::lib.schema.literal/literal]]])(mr/def ::filter-inside"Geographic bounding-box filter."- [:map+ [:map {:closed true}[:node/type [:= :filter/inside]][:lat-dimension ::dimension-or-expression][:lon-dimension ::dimension-or-expression]- [:north :any]- [:east :any]- [:south :any]- [:west :any]])+ [:north number?]+ [:east number?]+ [:south number?]+ [:west number?]])(mr/def ::filter-temporal"Temporal filter for time-based operations."- [:map+ [:map {:closed true}[:node/type [:= :filter/temporal]][:operator [:enum :time-interval :relative-time-interval]][:dimension ::dimension-or-expression]@@ -186,9 +195,9 @@(mr/def ::filter-mbql"Raw MBQL filter clause passthrough for source filters."- [:map+ [:map {:closed true}[:node/type [:= :filter/mbql]]- [:clause :any]])+ [:clause ::lib.schema.mbql-clause/clause]]);; Forward declare for recursive references(mr/def ::filter-node@@ -202,13 +211,13 @@::filter-in::filter-temporal::filter-mbql- [:map+ [:map {:closed true}[:node/type [:= :filter/and]][:children [:sequential [:ref ::filter-node]]]]- [:map+ [:map {:closed true}[:node/type [:= :filter/or]][:children [:sequential [:ref ::filter-node]]]]- [:map+ [:map {:closed true}[:node/type [:= :filter/not]][:child [:ref ::filter-node]]]])@@ -216,23 +225,23 @@(mr/def ::join-node"A join from the source metric's query, preserved as raw MBQL 5."- [:map+ [:map {:closed true}[:node/type [:= :ast/join]]- [:mbql-join :any]])+ [:mbql-join ::lib.schema.join/join]]);;; -------------------- Source Nodes --------------------(defn- source-node-schema"Create a source node schema with the given node-type keyword."[node-type]- [:map+ [:map {:closed true}[:node/type [:= node-type]][:id pos-int?][:name {:optional true} [:maybe string?]][:aggregation ::aggregation-node][:base-table ::table-node][:source-card-id {:optional true} [:maybe pos-int?]]- [:metadata {:optional true} [:maybe :map]]+ [:metadata {:optional true} [:maybe [:or ::lib.schema.metadata/metric ::lib.schema.metadata/measure]]][:joins {:optional true} [:maybe [:sequential ::join-node]]][:filters {:optional true} [:maybe [:ref ::filter-node]]]])@@ -253,7 +262,7 @@(mr/def ::source-query"A single-source query node with source, dimensions, mappings, filters, and group-by.Used inside expression leaves as the compilable sub-query."- [:map+ [:map {:closed true}[:node/type [:= :ast/source-query]][:source ::source-node][:dimensions [:sequential ::dimension-node]]diff --git a/src/metabase/lib_metric/db.clj b/src/metabase/lib_metric/db.cljindex 4be478cedc5..9ed565f6e38 100644--- a/src/metabase/lib_metric/db.clj+++ b/src/metabase/lib_metric/db.clj@@ -40,7 +40,8 @@(mu/defn metrics"The metric Cards (`:metadata/metric`) picked out by `metadata-spec`."- [metadata-spec :- [:map+ [metadata-spec :- [:map {:closed true}+ [:lib/type {:optional true} [:= :metadata/metric]][:id {:optional true} [:maybe [:set ms/PositiveInt]]][:name {:optional true} [:maybe [:set :string]]][:table-ids {:optional true} [:maybe [:set ::lib.schema.id/table]]]@@ -61,7 +62,8 @@(mu/defn measures"The `:metadata/measure` rows picked out by `metadata-spec`."- [metadata-spec :- [:map+ [metadata-spec :- [:map {:closed true}+ [:lib/type {:optional true} [:= :metadata/measure]][:id {:optional true} [:maybe [:set ms/PositiveInt]]][:name {:optional true} [:maybe [:set :string]]][:table-ids {:optional true} [:maybe [:set ::lib.schema.id/table]]]]]diff --git a/src/metabase/lib_metric/definition.cljc b/src/metabase/lib_metric/definition.cljcindex 5c27f7d3a83..40d3e2c62f3 100644--- a/src/metabase/lib_metric/definition.cljc+++ b/src/metabase/lib_metric/definition.cljc@@ -5,6 +5,8 @@[metabase.lib-metric.ast.plan :as ast.plan][metabase.lib-metric.ast.schema :as ast.schema][metabase.lib-metric.schema :as lib-metric.schema]+ [metabase.lib.metadata.protocols :as lib.metadata.protocols]+ [metabase.lib.schema.metadata :as lib.schema.metadata][metabase.util.malli :as mu][metabase.util.performance :as perf]))@@ -66,7 +68,8 @@(mu/defn from-metric-metadata :- ::lib-metric.schema/metric-definition"Create a MetricDefinition from MetricMetadata.Dimensions and dimension-mappings are derived from source metadata when building the AST."- [provider metric-metadata]+ [provider :- [:maybe ::lib.metadata.protocols/metadata-providerable]+ metric-metadata :- ::lib.schema.metadata/metric](let [uuid (str (random-uuid))]{:lib/type :metric/definition:expression [:metric {:lib/uuid uuid} (:id metric-metadata)]@@ -77,7 +80,8 @@(mu/defn from-measure-metadata :- ::lib-metric.schema/metric-definition"Create a MetricDefinition from MeasureMetadata.Dimensions and dimension-mappings are derived from source metadata when building the AST."- [provider measure-metadata]+ [provider :- [:maybe ::lib.metadata.protocols/metadata-providerable]+ measure-metadata :- ::lib.schema.metadata/measure](let [uuid (str (random-uuid))]{:lib/type :metric/definition:expression [:measure {:lib/uuid uuid} (:id measure-metadata)]@@ -157,6 +161,8 @@([definition :- ::lib-metric.schema/metric-definition](->query-plan definition {}))([definition :- ::lib-metric.schema/metric-definition- opts]+ opts :- [:map {:closed true}+ [:limit {:optional true} [:maybe pos-int?]]+ [:values-only {:optional true} [:maybe :boolean]]]](let [ast (->ast definition)](ast.plan/plan-from-ast ast opts))))diff --git a/src/metabase/lib_metric/dimension.cljc b/src/metabase/lib_metric/dimension.cljcindex c61220f7bb8..b71d163401a 100644--- a/src/metabase/lib_metric/dimension.cljc+++ b/src/metabase/lib_metric/dimension.cljc@@ -437,7 +437,7 @@(mu/defn get-dimension-or-throw :- :map"Find a dimension by its UUID from a list of dimensions.Throws 400 if not found."- [dimensions :- [:maybe [:sequential :map]]+ [dimensions :- [:maybe [:sequential ::lib-metric.schema/dimension]]dimension-id :- :string](or (m/find-first #(= (:id %) dimension-id) dimensions)(throw (ex-info (i18n/tru "Dimension not found: {0}" dimension-id)@@ -447,7 +447,7 @@(mu/defn get-dimension-mapping-or-throw :- :map"Find a dimension mapping by dimension UUID from a list of mappings.Throws 400 if not found."- [dimension-mappings :- [:maybe [:sequential :map]]+ [dimension-mappings :- [:maybe [:sequential ::lib-metric.schema/dimension-mapping]]dimension-id :- :string](or (m/find-first #(= (:dimension-id %) dimension-id) dimension-mappings)(throw (ex-info (i18n/tru "Dimension mapping not found for dimension: {0}" dimension-id)@@ -471,8 +471,8 @@Validates that the dimension is active (not orphaned) and has a valid mapping.Returns the field ID or throws an exception with appropriate error message."- [dimensions :- [:maybe [:sequential :map]]- dimension-mappings :- [:maybe [:sequential :map]]+ [dimensions :- [:maybe [:sequential ::lib-metric.schema/dimension]]+ dimension-mappings :- [:maybe [:sequential ::lib-metric.schema/dimension-mapping]]dimension-id :- :string](let [dimension (get-dimension-or-throw dimensions dimension-id)];; Check for orphaned dimensionsdiff --git a/src/metabase/lib_metric/metadata/provider.cljc b/src/metabase/lib_metric/metadata/provider.cljcindex 66706b3b481..218f4e57901 100644--- a/src/metabase/lib_metric/metadata/provider.cljc+++ b/src/metabase/lib_metric/metadata/provider.cljc@@ -178,16 +178,40 @@- Routes dimension requests to dimension-fetcher-fn if provided- Routes table/column/segment requests to the appropriate database provider- Caches metric, measure, and dimension metadata internally"- ([metric-fetcher-fn table->db-fn db-provider-fn setting-fn]+ ([metric-fetcher-fn :- ifn?+ table->db-fn :- ifn?+ db-provider-fn :- ifn?+ setting-fn :- ifn?](metric-context-metadata-provider metric-fetcher-fn nil nil nil table->db-fn db-provider-fn setting-fn nil))- ([metric-fetcher-fn measure-fetcher-fn table->db-fn db-provider-fn setting-fn]+ ([metric-fetcher-fn :- ifn?+ measure-fetcher-fn :- [:maybe ifn?]+ table->db-fn :- ifn?+ db-provider-fn :- ifn?+ setting-fn :- ifn?](metric-context-metadata-provider metric-fetcher-fn measure-fetcher-fn nil nil table->db-fn db-provider-fn setting-fn nil))- ([metric-fetcher-fn measure-fetcher-fn dimension-fetcher-fn table->db-fn db-provider-fn setting-fn]+ ([metric-fetcher-fn :- ifn?+ measure-fetcher-fn :- [:maybe ifn?]+ dimension-fetcher-fn :- [:maybe ifn?]+ table->db-fn :- ifn?+ db-provider-fn :- ifn?+ setting-fn :- ifn?](metric-context-metadata-provider metric-fetcher-fn measure-fetcher-fn dimension-fetcher-fn nil table->db-fn db-provider-fn setting-fn nil))- ([metric-fetcher-fn measure-fetcher-fn dimension-fetcher-fn table->db-fn db-provider-fn setting-fn column-post-process-fn]+ ([metric-fetcher-fn :- ifn?+ measure-fetcher-fn :- [:maybe ifn?]+ dimension-fetcher-fn :- [:maybe ifn?]+ table->db-fn :- ifn?+ db-provider-fn :- ifn?+ setting-fn :- ifn?+ column-post-process-fn :- [:maybe ifn?]](metric-context-metadata-provider metric-fetcher-fn measure-fetcher-fn dimension-fetcher-fn nil table->db-fn db-provider-fn setting-fn column-post-process-fn))- ([metric-fetcher-fn measure-fetcher-fn dimension-fetcher-fn segment-fetcher-fn- table->db-fn db-provider-fn setting-fn column-post-process-fn]+ ([metric-fetcher-fn :- ifn?+ measure-fetcher-fn :- [:maybe ifn?]+ dimension-fetcher-fn :- [:maybe ifn?]+ segment-fetcher-fn :- [:maybe ifn?]+ table->db-fn :- ifn?+ db-provider-fn :- ifn?+ setting-fn :- ifn?+ column-post-process-fn :- [:maybe ifn?]](->MetricContextMetadataProvidermetric-fetcher-fnmeasure-fetcher-fndiff --git a/src/metabase/lib_metric/schema.cljc b/src/metabase/lib_metric/schema.cljcindex a79f5543ba6..632f9ab8e50 100644--- a/src/metabase/lib_metric/schema.cljc+++ b/src/metabase/lib_metric/schema.cljc@@ -3,6 +3,7 @@(:refer-clojure :exclude [some])(:require[metabase.lib-metric.operators :as operators]+ [metabase.lib.metadata.protocols :as lib.metadata.protocols][metabase.lib.schema.common :as lib.schema.common][metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.literal :as lib.schema.literal]@@ -44,7 +45,7 @@(mr/def ::binning-option"Schema for a binning option as presented to the UI."- [:map+ [:map {:closed true}[:lib/type [:= :option/binning]][:display-name :string][:mbql [:maybe ::binning]]@@ -55,23 +56,16 @@(mr/def ::dimension-id"UUID string identifying a dimension."- ::lib.schema.common/uuid)+ ::lib.schema.metadata/dimension-id)(mr/def ::dimension-group- "Group descriptor for a dimension, indicating which table it belongs to."- [:map {:closed true}- [:id :string]- [:type [:enum "main" "connection"]]- [:display-name :string]])+ ::lib.schema.metadata/dimension-group)(mr/def ::dimension-source.type- [:enum :field])+ ::lib.schema.metadata/dimension-source.type)(mr/def ::dimension-source- [:map {:closed true}- [:type ::dimension-source.type]- [:field-id {:optional true} [:maybe ::lib.schema.id/field]]- [:binning {:optional true} [:maybe :boolean]]])+ ::lib.schema.metadata/dimension-source)(mr/def ::dimension"Schema for a dimension definition, plus what the metrics module annotates a dimension with before handing it to a@@ -94,12 +88,10 @@[:description {:optional true} [:maybe :string]]])(mr/def ::dimension-mapping.type- "Type of dimension mapping."- [:enum :table])+ ::lib.schema.metadata/dimension-mapping.type)(mr/def ::dimension-mapping.target- "Target field reference for a dimension mapping, e.g. [:field {:source-field 1} 2]."- [:ref :mbql.clause/field])+ ::lib.schema.metadata/dimension-mapping.target)(mr/def ::dimension-mapping"Schema for a dimension mapping."@@ -338,17 +330,11 @@;;; These schemas are used for storage format in the database.(mr/def ::dimension-status- "Status of a dimension indicating whether it's active or has issues.- - :status/active - Column exists, dimension is usable- - :status/orphaned - Column was removed from schema, dimension preserved for reference"- [:enum :status/active :status/orphaned])+ ::lib.schema.metadata/dimension-status)(mr/def ::persisted-dimension- "Schema for a persisted dimension definition with status tracking.- Persisted dimensions include additional metadata about their status- and any issues that prevent them from being used.- Note: target field references are stored in dimension-mappings, not here."- [:map+ "Schema for a persisted dimension definition with status tracking."+ [:map {:closed true}[:id ::dimension-id][:name {:optional true} [:maybe :string]][:display-name {:optional true} [:maybe ::lib.schema.common/non-blank-string]]@@ -360,8 +346,8 @@[:status-message {:optional true} [:maybe :string]][:sources {:optional true} [:maybe [:sequential ::dimension-source]]][:group {:optional true} [:maybe ::dimension-group]]+ [:lib/source {:optional true} [:maybe [:or ::lib.schema.metadata/column.source :string]]][:default-temporal-unit {:optional true} ::lib.schema.temporal-bucketing/unit]- ;; At most one dimension per entity may be the default.[:default {:optional true} [:maybe :boolean]]])(mr/def ::persisted-dimensions@@ -379,12 +365,12 @@Metadata is loaded lazily from the provider in the AST builder,not stored in the definition."- [:map+ [:map {:closed true}[:lib/type [:= :metric/definition]][:expression ::metric-math-expression][:filters ::instance-filters][:projections ::typed-projections]- [:metadata-provider [:maybe :some]]])+ [:metadata-provider [:maybe ::lib.metadata.protocols/metadata-providerable]]]);;; ------------------------------------------------- Fetchable Dimension Metadata -------------------------------------------------;;; These schemas support dimensions as first-class metadata entities@@ -398,11 +384,12 @@"Schema for dimension metadata fetchable via metadata provider.Dimensions are extracted from metrics/measures at fetch time, with sourcetracking to identify their parent entity."- [:map+ [:map {:closed true}[:lib/type [:= :metadata/dimension]][:id ::dimension-id] ; UUID string[:name {:optional true} [:maybe :string]][:display-name {:optional true} [:maybe ::lib.schema.common/non-blank-string]]+ [:description {:optional true} [:maybe :string]][:effective-type {:optional true} [:maybe ::lib.schema.common/base-type]][:semantic-type {:optional true} [:maybe ::lib.schema.common/semantic-or-relation-type]][:has-field-values {:optional true} [:maybe [:enum :list :search :none]]]@@ -410,13 +397,15 @@[:status-message {:optional true} [:maybe :string]][:sources {:optional true} [:maybe [:sequential ::dimension-source]]][:group {:optional true} [:maybe ::dimension-group]]+ [:lib/source {:optional true} [:maybe [:or ::lib.schema.metadata/column.source :string]]][:default-temporal-unit {:optional true} ::lib.schema.temporal-bucketing/unit][:default {:optional true} [:maybe :boolean]];; Source tracking[:source-type ::dimension-source-type][:source-id pos-int?];; Optional mapping for field resolution- [:dimension-mapping {:optional true} [:maybe ::dimension-mapping]]])+ [:dimension-mapping {:optional true} [:maybe ::dimension-mapping]]+ [:projection-positions {:optional true} [:maybe [:sequential :int]]]])(mr/def ::dimension-spec"Spec for fetching dimensions from metadata provider.@@ -434,7 +423,7 @@(mr/def ::computed-dimension"A dimension computed from a visible column, before reconciliation.The :id is nil until assigned during reconciliation."- [:map+ [:map {:closed true}[:id [:maybe ::dimension-id]][:name :string][:display-name {:optional true} [:maybe :string]]@@ -442,13 +431,14 @@[:semantic-type {:optional true} [:maybe :keyword]][:has-field-values {:optional true} [:maybe [:enum :list :search :none]]][:lib/source {:optional true} [:maybe :keyword]]+ [:sources {:optional true} [:maybe [:sequential ::dimension-source]]][:group {:optional true} [:maybe ::dimension-group]]])(mr/def ::computed-pair"A computed dimension paired with its mapping (before ID assignment)."- [:map+ [:map {:closed true}[:dimension ::computed-dimension]- [:mapping [:map+ [:mapping [:map {:closed true}[:type ::dimension-mapping.type][:table-id {:optional true} [:maybe ::lib.schema.id/table]][:target ::dimension-mapping.target]]]])diff --git a/src/metabase/logger/api.clj b/src/metabase/logger/api.cljindex 748ff43ae1a..77d2cc68c9f 100644--- a/src/metabase/logger/api.clj+++ b/src/metabase/logger/api.clj@@ -124,7 +124,7 @@[:sequential ::log-adjustment])(mu/defn- set-log-levels! :- ::plan- [log-levels]+ [log-levels :- (ms/string-keyed-map ::log-level)](let [plan (create-plan log-levels)](run! execute-plan! plan)plan))diff --git a/src/metabase/login_history/record.clj b/src/metabase/login_history/record.cljindex 51289f34271..22d07fd5f6d 100644--- a/src/metabase/login_history/record.clj+++ b/src/metabase/login_history/record.clj@@ -32,10 +32,7 @@(mu/defn record-login-history!"Record login history for a user, and send them an email if this is their first time logging in from this device."[session-id :- string?- user :- [:map- {:description ":model/User"}- [:id pos-int?]- [:last_login {:optional true} :any]]+ user :- :metabase.users.schema/userdevice-info :- request/DeviceInfo](let [history-entry (login-history/record-login-history! session-id (u/the-id user) device-info)](when-not (or (:embedded device-info) (:token_exchange device-info))diff --git a/src/metabase/login_history/schema.clj b/src/metabase/login_history/schema.cljindex ae99336442e..93565609cd1 100644--- a/src/metabase/login_history/schema.clj+++ b/src/metabase/login_history/schema.clj@@ -7,15 +7,11 @@(mr/def ::login-history"A LoginHistory as selected from the app DB: every column of `:login_history`, plus `:active` added by the model's after-select hook."- [:map {:closed true}- [:id ms/PositiveInt]- [:timestamp ms/TemporalInstant]- [:user_id ::lib.schema.id/user]- [:session_id {:optional true} [:maybe :string]]- [:device_id :string]- [:device_description :string]- [:ip_address :string]- [:active {:optional true} :boolean]])+ [:merge+ ::login-history.update+ [:map {:closed true}+ [:id ms/PositiveInt]+ [:active {:optional true} :boolean]]])(mr/def ::login-history.update"What an update (or insert) of a LoginHistory accepts: every column of `:login_history` except `id`, all optional."diff --git a/src/metabase/mcp/schema.clj b/src/metabase/mcp/schema.cljindex 2d64d3bfa56..79e47e8f593 100644--- a/src/metabase/mcp/schema.clj+++ b/src/metabase/mcp/schema.clj@@ -7,15 +7,10 @@(mr/def ::mcp-feedback"A McpFeedback as selected from the app DB: every column of `:mcp_feedback`."- [:map {:closed true}- [:id ms/PositiveInt]- [:user_id ::lib.schema.id/user]- [:positive :boolean]- [:issue_type [:maybe [:or :keyword :string]]]- [:freeform_feedback [:maybe :string]]- [:prompt [:maybe :string]]- [:query [:maybe :string]]- [:created_at ms/TemporalInstant]])+ [:merge+ ::mcp-feedback.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::mcp-feedback.update"What an update (or insert) of a McpFeedback accepts: every column of `:mcp_feedback` except `id`, all optional."@@ -30,13 +25,10 @@(mr/def ::mcp-query-handle"A McpQueryHandle as selected from the app DB: every column of `:mcp_query_handle`."- [:map {:closed true}- [:id :string]- [:mcp_session_id :string]- [:core_session_id [:maybe :string]]- [:encoded_query :string]- [:created_at ms/TemporalInstant]- [:prompt [:maybe :string]]])+ [:merge+ ::mcp-query-handle.update+ [:map {:closed true}+ [:id :string]]])(mr/def ::mcp-query-handle.update"What an update (or insert) of a McpQueryHandle accepts: every column of `:mcp_query_handle` except `id`, all optional."@@ -49,16 +41,10 @@(mr/def ::mcp-session-log"A McpSessionLog as selected from the app DB: every column of `:mcp_session_log`."- [:map {:closed true}- [:id :string]- [:created_at ms/TemporalInstant]- [:ended_at [:maybe ms/TemporalInstant]]- [:user_id [:maybe ::lib.schema.id/user]]- [:tenant_id [:maybe ms/PositiveInt]]- [:client_name [:maybe :string]]- [:client_version [:maybe :string]]- [:ip_address [:maybe :string]]- [:user_agent [:maybe :string]]])+ [:merge+ ::mcp-session-log.update+ [:map {:closed true}+ [:id :string]]])(mr/def ::mcp-session-log.update"What an update (or insert) of a McpSessionLog accepts: every column of `:mcp_session_log` except `id`, all optional."@@ -74,21 +60,10 @@(mr/def ::mcp-tool-call-log"A McpToolCallLog as selected from the app DB: every column of `:mcp_tool_call_log`."- [:map {:closed true}- [:id ms/PositiveInt]- [:created_at ms/TemporalInstant]- [:user_id [:maybe ::lib.schema.id/user]]- [:tool_name :string]- [:status [:maybe [:or :keyword :string]]]- [:duration_ms [:maybe :int]]- [:error_code [:maybe :int]]- [:error_message [:maybe :string]]- [:client_name [:maybe :string]]- [:client_version [:maybe :string]]- [:tenant_id [:maybe ms/PositiveInt]]- [:ip_address [:maybe :string]]- [:user_agent [:maybe :string]]- [:sanitized_user_agent [:maybe :string]]])+ [:merge+ ::mcp-tool-call-log.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::mcp-tool-call-log.update"What an update (or insert) of a McpToolCallLog accepts: every column of `:mcp_tool_call_log` except `id`, all optional."diff --git a/src/metabase/mcp/v2/projections.clj b/src/metabase/mcp/v2/projections.cljindex 47ef4f095cd..fa34a3c2b66 100644--- a/src/metabase/mcp/v2/projections.clj+++ b/src/metabase/mcp/v2/projections.clj@@ -16,7 +16,6 @@(:require[clojure.string :as str][medley.core :as m]- [metabase.parameters.params :as params][metabase.util :as u]))(set! *warn-on-reflection* true)@@ -252,9 +251,14 @@(defn- dashboard-parameters-summary"One row per dashboard parameter: id, name, type, and the ids of the dashcards it is wired- to (via [[params/dashboard->resolved-params]], the same resolution the QP uses)."+ to through their `:parameter_mappings`, including the negative ids of a dry run's unsaved dashcards."[dash]- (let [resolved (params/dashboard->resolved-params dash)]+ (let [param-id->dashcard-ids (reduce (fn [acc {dashcard-id :id :keys [parameter_mappings]}]+ (reduce #(update %1 (u/qualified-name (:parameter_id %2)) (fnil conj #{}) dashcard-id)+ acc+ parameter_mappings))+ {}+ (:dashcards dash))](mapv (fn [{param-id :id :as param}](m/remove-valsnil?@@ -262,8 +266,7 @@:name (:name param):type (:type param):dashcard_ids (not-empty- (vec (sort (distinct (keep (comp :id :dashcard)- (:mappings (get resolved (u/qualified-name param-id))))))))}))+ (vec (sort (remove nil? (get param-id->dashcard-ids (u/qualified-name param-id))))))}))(:parameters dash))))(defn dashboard-rowdiff --git a/src/metabase/mcp/v2/resources.clj b/src/metabase/mcp/v2/resources.cljindex 3a45ad95935..9e29e2e9909 100644--- a/src/metabase/mcp/v2/resources.clj+++ b/src/metabase/mcp/v2/resources.clj@@ -29,7 +29,7 @@"Register a v2 MCP data resource, returning its URI. Overwrites any existing entry with thesame `:uri`. `:render-fn` produces the resource text at read time, so the content is alwayscurrent rather than a snapshot from registration."- [resource :- [:map+ [resource :- [:map {:closed true}[:uri :string][:name :string][:description :string]@@ -42,7 +42,7 @@(mu/defn register-ui-resource!"Register a v2 MCP Apps UI resource, returning its URI. Overwrites any existing entry with thesame `:uri`."- [resource :- [:map+ [resource :- [:map {:closed true}[:uri :string][:name :string][:description :string]diff --git a/src/metabase/mcp/v2/tools/dashboard.clj b/src/metabase/mcp/v2/tools/dashboard.cljindex ea634ffc3d2..c81243f61d3 100644--- a/src/metabase/mcp/v2/tools/dashboard.clj+++ b/src/metabase/mcp/v2/tools/dashboard.clj@@ -17,6 +17,7 @@(:require[malli.error :as me][metabase.dashboards.write :as dashboards.write]+ [metabase.lib.core :as lib][metabase.mcp.db :as mcp.db][metabase.mcp.v2.common :as common][metabase.mcp.v2.dashboard-ops :as dashboard-ops]@@ -27,7 +28,6 @@[metabase.mcp.v2.write :as v2.write][metabase.metabot.scope :as metabot.scope][metabase.models.interface :as mi]- [metabase.parameters.core :as parameters][metabase.util :as u][metabase.util.malli.registry :as mr][toucan2.core :as t2]))@@ -113,7 +113,7 @@[payload](cond-> payload(seq (:parameters payload))- (update :parameters #(try (parameters/normalize-parameters %) (catch Exception _ %)))))+ (update :parameters #(try (lib/normalize :metabase.parameters.schema/parameters %) (catch Exception _ %)))))(defn- validate-payload!"Reject a payload the real save would reject, so a dry run is worth trusting."diff --git a/src/metabase/mcp/v2/tools/question.clj b/src/metabase/mcp/v2/tools/question.cljindex e21afadb0a4..a497900c8ee 100644--- a/src/metabase/mcp/v2/tools/question.clj+++ b/src/metabase/mcp/v2/tools/question.clj@@ -386,7 +386,7 @@the dashboard whenever `:dashboard_id` is present, independent of `:archived`."[card-before card-updates](cond-> card-updates- (and (api/column-will-change? :dashboard_id card-before card-updates)+ (and (api/column-will-change? (:dashboard_id card-before) (get card-updates :dashboard_id ::api/not-provided))(:dashboard_id card-updates))(as-> updates(dodiff --git a/src/metabase/mcp/v2/tools/subscription.clj b/src/metabase/mcp/v2/tools/subscription.cljindex b4454edd101..1233a082819 100644--- a/src/metabase/mcp/v2/tools/subscription.clj+++ b/src/metabase/mcp/v2/tools/subscription.clj@@ -269,7 +269,10 @@:when (int? (:id card))](-> (api/read-check card)(select-keys [:id :name :collection_id :description :display :parameter_mappings])- (assoc :dashboard_card_id dashcard-id :dashboard_id (:id dashboard))))]+ (assoc :dashboard_card_id dashcard-id+ :dashboard_id (:id dashboard)+ :include_csv false+ :include_xls false)))](when (empty? cards)(common/throw-teaching-error"This dashboard has no cards to send — a subscription needs at least one saved question, model, or metric on the dashboard."))diff --git a/src/metabase/mcp/v2/tools/transform.clj b/src/metabase/mcp/v2/tools/transform.cljindex 253d3ff2d6d..cd57f137d6d 100644--- a/src/metabase/mcp/v2/tools/transform.clj+++ b/src/metabase/mcp/v2/tools/transform.clj@@ -126,7 +126,7 @@session-id api/*current-user-id* query_handle):querynormalize-transform-query))]- {:type "query" :query query}))+ {:type :query :query query}));;; ----------------------------------------------- Target handling ------------------------------------------------diff --git a/src/metabase/measures/db.clj b/src/metabase/measures/db.cljindex dfc2258a4ae..8d2516c2907 100644--- a/src/metabase/measures/db.clj+++ b/src/metabase/measures/db.clj@@ -44,7 +44,7 @@(mu/defn table"The Table with `table-id`, or nil."- [table-id :- ::lib.schema.id/table]+ [table-id :- [:maybe ::lib.schema.id/table]](t2/select-one :model/Table :id table-id {:from [(warehouse-schema-overlay/table-query)]}))(mu/defn table-perms-columns@@ -61,5 +61,5 @@"Set the dimensions and dimension mappings of the Measure with `id`."[id :- ::lib.schema.id/measuredimensions :- [:maybe sequential?]- dimension-mappings :- [:maybe [:sequential :map]]]+ dimension-mappings :- [:maybe [:sequential ::measures.schema/measure.dimension-mapping]]](t2/update! :model/Measure id {:dimensions dimensions, :dimension_mappings dimension-mappings}))diff --git a/src/metabase/measures/models/measure.clj b/src/metabase/measures/models/measure.cljindex 90167713c7c..7cf8812953f 100644--- a/src/metabase/measures/models/measure.clj+++ b/src/metabase/measures/models/measure.clj@@ -10,6 +10,7 @@[metabase.lib.schema.common :as lib.schema.common][metabase.lib.schema.measure :as lib.schema.measure][metabase.measures.db :as measures.db]+ [metabase.measures.schema][metabase.metrics.core :as metrics][metabase.models.interface :as mi][metabase.models.serialization :as serdes]@@ -33,7 +34,8 @@Throws an exception with 'Invalid measure definition' if the definition is not valid MBQL5."[definition](when (seq definition)- (when-not (= :mbql-version/mbql5 (lib/normalized-mbql-version definition))+ (when-not (= :mbql-version/mbql5 (when (map? definition)+ (u/ignore-exceptions (lib/normalized-mbql-version definition))))(throw (ex-info (tru "Invalid measure definition: expected MBQL5 format"){:definition definition})))(mu/validate-throw ::lib.schema.measure/definition definition)))diff --git a/src/metabase/measures/schema.clj b/src/metabase/measures/schema.cljindex f20e73992be..92ba1df1888 100644--- a/src/metabase/measures/schema.clj+++ b/src/metabase/measures/schema.clj@@ -1,11 +1,14 @@(ns metabase.measures.schema(:require[metabase.lib-be.schema :as lib-be.schema]+ [metabase.lib-metric.schema :as lib-metric.schema][metabase.lib.core :as lib][metabase.lib.schema.id :as lib.schema.id]+ [metabase.users.schema][metabase.util.i18n :refer [deferred-tru]][metabase.util.malli.registry :as mr]- [metabase.util.malli.schema :as ms]))+ [metabase.util.malli.schema :as ms]+ [metabase.warehouse-schema.schema]))(set! *warn-on-reflection* true)@@ -21,31 +24,25 @@(mr/def ::measure.definition"The `:definition` column of a Measure, decoded."- :map)+ ::lib-be.schema/maybe-legacy-query)(mr/def ::measure.dimension"One entry of the `:dimensions` column of a Measure, decoded."- :map)+ ::lib-metric.schema/persisted-dimension)(mr/def ::measure.dimension-mapping"One entry of the `:dimension_mappings` column of a Measure, decoded."- :map)+ ::lib-metric.schema/dimension-mapping)(mr/def ::measure- "A Measure as selected from the app DB: every column of `:measure`."- [:map {:closed true}- [:id ms/PositiveInt]- [:table_id ::lib.schema.id/table]- [:creator_id ::lib.schema.id/user]- [:name :string]- [:description [:maybe :string]]- [:archived :boolean]- [:definition ::measure.definition]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]- [:entity_id :string]- [:dimensions [:maybe [:sequential ::measure.dimension]]]- [:dimension_mappings [:maybe [:sequential ::measure.dimension-mapping]]]])+ "A Measure as selected from the app DB: every column of `:measure`, plus `:creator` and `:table` some callers+ hydrate onto it."+ [:merge+ ::measure.update+ [:map {:closed true}+ [:id ms/PositiveInt]+ [:creator {:optional true} [:maybe :metabase.users.schema/user]]+ [:table {:optional true} [:maybe :metabase.warehouse-schema.schema/table]]]])(mr/def ::measure.update"What an update (or insert) of a Measure accepts: every column of `:measure` except `id`, all optional."diff --git a/src/metabase/metabot/agent/core.clj b/src/metabase/metabot/agent/core.cljindex 94c3f697fb2..e6eb56cd812 100644--- a/src/metabase/metabot/agent/core.clj+++ b/src/metabase/metabot/agent/core.clj@@ -16,6 +16,7 @@[metabase.metabot.agent.profiles :as profiles][metabase.metabot.agent.streaming :as streaming][metabase.metabot.capabilities :as capabilities]+ [metabase.metabot.context :as metabot.context][metabase.metabot.metadata-perms :as metabot.perms][metabase.metabot.schema :as metabot.schema][metabase.metabot.scope :as scope]@@ -95,7 +96,7 @@(mr/def ::content-block"A content block in a multi-part message (Claude format, backward-compat)."- [:map+ [:map {:closed true}[:type :string]]) ;; "text", "tool_use", "tool_result", etc.(mr/def ::content@@ -104,36 +105,36 @@(mr/def ::tool-call"A tool call in an assistant message."- [:map+ [:map {:closed true}[:id :string][:name :string]- [:arguments [:or :string :map]]])+ [:arguments :string]])(mr/def ::user-message"A user message: plain text or a sequence of tool_result content blocks."- [:map+ [:map {:closed true}[:role [:= :user]][:content ::content]])(mr/def ::assistant-message"An assistant message with optional text and/or tool calls."- [:map+ [:map {:closed true}[:role [:= :assistant]][:content {:optional true} [:maybe ::content]][:tool_calls {:optional true} [:maybe [:sequential ::tool-call]]]])(mr/def ::system-message"A system message."- [:map+ [:map {:closed true}[:role [:= :system]][:content :string]])(mr/def ::tool-message"A tool result message, referencing a previous tool call by ID."- [:map+ [:map {:closed true}[:role [:= :tool]][:tool_call_id :string]- [:content [:or :string :map]]])+ [:content :string]])(mr/def ::message"A single message in the conversation history.@@ -150,7 +151,7 @@(mr/def ::context"Context information for the agent."- [:map-of :keyword :any])+ ::metabot.context/context)(mr/def ::profile-id"Profile identifier keyword."@@ -158,7 +159,7 @@(mr/def ::tracking-opts"Options for snowplow and prometheus analytics tracking."- [:map+ [:map {:closed true}[:session-id {:optional true} [:maybe ms/UUIDString]][:source {:optional true} [:maybe :string]][:tag {:optional true} [:maybe :string]]])@@ -686,7 +687,7 @@(into [] (run-agent-loop opts))(transduce xf rf (run-agent-loop opts))(into [] (run-agent-loop (assoc opts :debug? true))) ;; with debug log"- [opts :- [:map+ [opts :- [:map {:closed true}[:messages ::messages][:profile-id ::profile-id][:metabot-id {:optional true} [:maybe :string]]diff --git a/src/metabase/metabot/agent/messages.clj b/src/metabase/metabot/agent/messages.cljindex 1ba2e1f5341..697cc3a56c0 100644--- a/src/metabase/metabot/agent/messages.clj+++ b/src/metabase/metabot/agent/messages.clj@@ -31,7 +31,7 @@[arguments](if (string? arguments)(try- (json/decode+kw arguments)+ (json/decode arguments)(catch Exception e(log/warnf "Failed to decode tool call arguments: %s" (ex-message e))arguments))diff --git a/src/metabase/metabot/agent/profiles.clj b/src/metabase/metabot/agent/profiles.cljindex 5299425f03b..c1b03046c93 100644--- a/src/metabase/metabot/agent/profiles.clj+++ b/src/metabase/metabot/agent/profiles.clj@@ -74,13 +74,15 @@Tool vars are validated at registration time to ensure they have required metadata; any`:always-on-skills` are validated to refer to registered skills, and any `:terminal-tools` torefer to tools the profile actually exposes."- [profile :- [:map+ [profile :- [:map {:closed true}[:name :keyword][:prompt-template :string][:max-iterations :int]- [:tools [:vector :any]]+ [:temperature {:optional true} [:maybe number?]]+ [:tools [:vector tool-var-schema]][:always-on-skills {:optional true} [:vector :keyword]][:skills? {:optional true} :boolean]+ [:required-tool-call? {:optional true} :boolean][:terminal-tools {:optional true} [:set :string]][:system-prompt-context {:optional true} [:fn ifn?]]]](let [tool-vars (:tools profile)diff --git a/src/metabase/metabot/api.clj b/src/metabase/metabot/api.cljindex 4d4b5b13e2b..f859b438f6c 100644--- a/src/metabase/metabot/api.clj+++ b/src/metabase/metabot/api.clj@@ -375,7 +375,7 @@(mu/defn- upgrade-viewing-queries"Update queries of items in viewing context vector. Handles following item types: adhoc, question, model, metric"- [viewing :- [:vector :map]]+ [viewing :- [:vector metabot.context/ViewingItemSchema]](letfn [(update-items-query [item] (m/update-existing item :query legacy->modern-query))(maybe-update-item [item] (cond-> item(contains? upgradable-item-types (:type item))diff --git a/src/metabase/metabot/context.clj b/src/metabase/metabot/context.cljindex 016ebcf6cdf..4ab84971407 100644--- a/src/metabase/metabot/context.clj+++ b/src/metabase/metabot/context.clj@@ -396,7 +396,8 @@[:slack_channel_id {:optional true} [:maybe :string]][:default_database_id {:optional true} [:maybe :int]][:code_editor {:optional true} [:maybe CodeEditorContextSchema]]- [:research_plan {:optional true} [:maybe ::research-plan]]])+ [:research_plan {:optional true} [:maybe ::research-plan]]+ [:references {:optional true} [:maybe ms/OpaqueJSONObject]]])(defn- query-for-sql-parsing"Given an item in context, return the query if it is a native query or SQL transform that can have table usage parsed@@ -619,7 +620,10 @@([context :- ::context](create-context context nil))([context :- ::context- opts :- [:maybe [:map-of :keyword :any]]]+ opts :- [:maybe [:map {:closed true}+ [:metabot-id {:optional true} [:maybe :string]]+ [:profile-id {:optional true} [:maybe :keyword]]+ [:date-format {:optional true} [:maybe (ms/InstanceOfClass DateTimeFormatter)]]]]](metabot.perms/with-cache(-> contextenhance-context-with-schemadiff --git a/src/metabase/metabot/conversation_title.clj b/src/metabase/metabot/conversation_title.cljindex 7016e806002..3a1105d17a3 100644--- a/src/metabase/metabot/conversation_title.clj+++ b/src/metabase/metabot/conversation_title.clj@@ -44,8 +44,8 @@(def ^:private title-json-schema{:type "object"- :properties {:title {:type "string"- :description "A short title for the conversation. No quotes or extra explanation."}}+ :properties {"title" {:type "string"+ :description "A short title for the conversation. No quotes or extra explanation."}}:required ["title"]:additionalProperties false})diff --git a/src/metabase/metabot/db.clj b/src/metabase/metabot/db.cljindex 324e0451281..e0245d004b7 100644--- a/src/metabase/metabot/db.clj+++ b/src/metabase/metabot/db.clj@@ -939,7 +939,7 @@archived set, and hardcoding `false` here drops every archived hit while the caller's totalstill counts it."[document-ids :- [:set ms/PositiveInt]- archived? :- :any]+ archived? :- :boolean](t2/select :model/Document :id [:in document-ids] :archived (boolean archived?)))(mu/defn transforms@@ -989,9 +989,10 @@:limit limit}))(mu/defn user-summary- "The ID, email, and names of the User with `user-id`."- [user-id :- ::lib.schema.id/user]- (t2/select-one [:model/User :id :email :first_name :last_name] user-id))+ "The ID, email, and names of the User with `user-id`, or nil if `user-id` is nil or matches no User."+ [user-id :- [:maybe ::lib.schema.id/user]]+ (when user-id+ (t2/select-one [:model/User :id :email :first_name :last_name] user-id)))(mu/defn user-summaries-by-id"A map of ID to the ID, email, and names of the Users with `user-ids`."diff --git a/src/metabase/metabot/example_question_generator.clj b/src/metabase/metabot/example_question_generator.cljindex c881a144461..574d5398d31 100644--- a/src/metabase/metabot/example_question_generator.clj+++ b/src/metabase/metabot/example_question_generator.clj@@ -74,9 +74,9 @@(def ^:private questions-json-schema"JSON Schema for the structured output tool, matching Python's ExampleQuestionGenerationSchema."{:type "object"- :properties {:questions {:type "array"- :items {:type "string"}- :description "List of example questions. No additional explanation."}}+ :properties {"questions" {:type "array"+ :items {:type "string"}+ :description "List of example questions. No additional explanation."}}:required ["questions"]:additionalProperties false})diff --git a/src/metabase/metabot/models/metabot.clj b/src/metabase/metabot/models/metabot.cljindex de52a0ae2ea..10f2a077808 100644--- a/src/metabase/metabot/models/metabot.clj+++ b/src/metabase/metabot/models/metabot.clj@@ -1,6 +1,7 @@(ns metabase.metabot.models.metabot(:require[metabase.metabot.db :as metabot.db]+ [metabase.metabot.schema][metabase.models.interface :as mi][metabase.models.serialization :as serdes][methodical.core :as methodical]diff --git a/src/metabase/metabot/models/metabot_message.clj b/src/metabase/metabot/models/metabot_message.cljindex f474d1fb658..2be534214bd 100644--- a/src/metabase/metabot/models/metabot_message.clj+++ b/src/metabase/metabot/models/metabot_message.clj@@ -1,6 +1,7 @@(ns metabase.metabot.models.metabot-message(:require[metabase.config.core :as config]+ [metabase.metabot.schema :as metabot.schema][metabase.metabot.schema.migrate-v1-to-v2 :as migrate][metabase.metabot.schema.v2 :as schema.v2][metabase.models.interface :as mi]@@ -16,9 +17,11 @@(derive :metabase/model))(t2/deftransforms :model/MetabotMessage- {:usage mi/transform-json+ {:usage {:in mi/json-in+ :out (comp metabot.schema/normalize-usage mi/json-out-with-keywordization)}:data mi/transform-json- :state mi/transform-json+ :state {:in mi/json-in+ :out (comp metabot.schema/normalize-state mi/json-out-with-keywordization)}:role mi/transform-keyword})(defn- migrate-v1->v2-on-readdiff --git a/src/metabase/metabot/models/metabot_prompt.clj b/src/metabase/metabot/models/metabot_prompt.cljindex 6a29dde8a5b..af69d0f8535 100644--- a/src/metabase/metabot/models/metabot_prompt.clj+++ b/src/metabase/metabot/models/metabot_prompt.clj@@ -1,6 +1,7 @@(ns metabase.metabot.models.metabot-prompt(:require[metabase.metabot.db :as metabot.db]+ [metabase.metabot.schema][metabase.models.interface :as mi][metabase.models.serialization :as serdes][methodical.core :as methodical]diff --git a/src/metabase/metabot/persistence.clj b/src/metabase/metabot/persistence.cljindex 31edfe9afd8..4c4c87def9e 100644--- a/src/metabase/metabot/persistence.clj+++ b/src/metabase/metabot/persistence.clj@@ -536,7 +536,7 @@`metabot_message` `messages` (in reader order, e.g. from [[live-messages]]):errored and in-flight turns are dropped, aborted turns replay their partialcontent."- [messages :- [:sequential :map]]+ [messages :- [:sequential ::metabot.schema/metabot-message]](into [](mapcat turn->llm-messages)(rows->turns messages)))diff --git a/src/metabase/metabot/reactions.clj b/src/metabase/metabot/reactions.cljindex a92a517d9ae..edd51e4a849 100644--- a/src/metabase/metabot/reactions.clj+++ b/src/metabase/metabot/reactions.clj@@ -38,7 +38,7 @@The schema name matches the reaction-name."[reaction-name :- ::reaction-type- schema]+ schema :- [:and :any [:fn {:error/message "a malli schema"} mc/schema]]](derive! reaction-name :metabot/registered-action)(mr/register! reaction-name schema))diff --git a/src/metabase/metabot/schema.clj b/src/metabase/metabot/schema.cljindex 8c34f3d283d..7bbbc5348ec 100644--- a/src/metabase/metabot/schema.clj+++ b/src/metabase/metabot/schema.clj@@ -4,6 +4,8 @@[malli.transform :as mtx][metabase.lib.schema.common :as lib.schema.common][metabase.lib.schema.id :as lib.schema.id]+ [metabase.lib.util :as lib.util]+ [metabase.metabot.schema.v2 :as schema.v2][metabase.util :as u][metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms]))@@ -29,17 +31,124 @@(mr/def ::state-map-key"A dynamic state-map key, normalized to its canonical string representation."- [:or {:decode/normalize lib.schema.common/normalize-string-key}- :string- :keyword])+ [:string {:decode/normalize lib.schema.common/normalize-string-key}])++(mr/def ::query+ "An MBQL 5 or legacy query in any state of normalization, as it round-trips through tool-call JSON or persisted turn state."+ ::lib.util/query-like)++(mr/def ::todo+ "One todo item of persisted turn state; only `:id` is guaranteed, since state can hold a partial item+ between the turn that creates it and the turn that fills in its details."+ [:map {:closed true}+ [:id :string]+ [:content {:optional true} [:maybe :string]]+ [:status {:optional true} [:maybe [:enum "pending" "in_progress" "completed" "cancelled"]]]+ [:priority {:optional true} [:maybe [:enum "high" "medium" "low"]]]])++(mr/def ::chart-timeline-event+ [:map {:closed true}+ [:name :string]+ [:description {:optional true} [:maybe :string]]+ [:timestamp :string]])++(mr/def ::column-info+ "A chart column's name and inferred type, as sent for chart analysis. Mirrors+ `metabase.metabot.context/ColumnInfoSchema`."+ [:map {:closed true}+ [:name :string]+ [:type {:optional true} [:maybe (into [:enum] #{"number" "string" "date" "datetime" "time" "boolean" "null"})]]])++(mr/def ::row-value+ "One cell value in a chart series. The `object` arm is never read by this code -- it's forwarded to the+ interestingness stats/repr code as-is -- so it's opaque rather than typed. Mirrors+ `metabase.metabot.context/RowValueSchema`."+ [:maybe [:or :string number? :boolean ms/OpaqueJSONObject]])++(mr/def ::chart-data+ "One pre-materialized table of raw chart data (columns + rows). Mirrors+ `metabase.metabot.context/ChartDataSchema`."+ [:map {:closed true}+ [:columns [:sequential ::column-info]]+ [:rows [:sequential [:sequential [:or :string number?]]]]])++(mr/def ::series-config+ "One series of a chart, pre-materialized by the frontend for `analyze_chart`. Mirrors+ `metabase.metabot.context/SeriesConfigSchema`."+ [:map {:closed true}+ [:x ::column-info]+ [:y {:optional true} [:maybe ::column-info]]+ [:x_values {:optional true} [:maybe [:sequential ::row-value]]]+ [:y_values {:optional true} [:maybe [:sequential ::row-value]]]+ [:display_name :string]+ [:chart_type [:or :string :keyword]]+ [:stacked {:optional true} [:maybe :boolean]]])++(mr/def ::chart-config+ "A `chart_configs` entry: a chart's title, pre-materialized series data, and the query that produced it.+ Mirrors `metabase.metabot.context/ChartConfigSchema`."+ [:map {:closed true}+ [:title {:optional true} [:maybe :string]]+ [:description {:optional true} [:maybe :string]]+ [:data {:optional true} [:maybe [:sequential ::chart-data]]]+ [:series {:optional true} [:maybe (ms/string-keyed-map ::series-config)]]+ [:timeline_events {:optional true} [:maybe [:sequential ::chart-timeline-event]]]+ [:query {:optional true} [:maybe ::query]]+ [:display_type {:optional true} [:maybe [:or :string :keyword]]]])++(mr/def ::chart+ [:map {:closed true}+ [:chart_id {:optional true} [:maybe :string]]+ [:query_id {:optional true} [:maybe :string]]+ [:queries {:optional true} [:maybe [:sequential [:maybe ::query]]]]+ [:visualization_settings {:optional true}+ [:maybe [:map {:closed true}+ [:chart_type {:optional true} [:maybe [:or :string :keyword]]]]]]+ [:timeline_events {:optional true} [:maybe [:sequential ::chart-timeline-event]]]+ [:chart_config {:optional true} [:maybe ::chart-config]]])++(mr/def ::transform.target+ [:map {:closed true}+ [:type [:or [:= :table] [:= "table"]]]+ [:name {:optional true} [:maybe :string]]+ [:database {:optional true} [:maybe :int]]+ [:schema {:optional true} [:maybe :string]]])++(mr/def ::transform.source-table+ "One entry of a Python transform's `source-tables`. Mirrors+ `metabase.metabot.context/TransformSourceTableSchema`."+ [:map {:closed true}+ [:alias :string]+ [:table_id {:optional true} [:maybe :int]]+ [:schema {:optional true} [:maybe :string]]+ [:database_id {:optional true} [:maybe :int]]])++(mr/def ::transform.source+ [:multi {:dispatch (comp keyword :type)}+ [:query [:map {:closed true}+ [:type [:or [:= :query] [:= "query"]]]+ [:query {:optional true} [:maybe ::query]]]]+ [:python [:map {:closed true}+ [:type [:or [:= :python] [:= "python"]]]+ [:body {:optional true} [:maybe :string]]+ [:source-database {:optional true} [:maybe :int]]+ [:source-tables {:optional true} [:maybe [:sequential ::transform.source-table]]]]]])++(mr/def ::transform+ [:map {:closed true}+ [:id {:optional true} [:maybe :string]]+ [:name {:optional true} [:maybe :string]]+ [:description {:optional true} [:maybe :string]]+ [:target {:optional true} [:maybe ::transform.target]]+ [:source {:optional true} [:maybe ::transform.source]]])(mr/def ::state- [:map- [:queries {:optional true} [:map-of ::state-map-key :map]]- [:charts {:optional true} [:map-of ::state-map-key :map]]- [:chart-configs {:optional true} [:map-of ::state-map-key :map]]- [:todos {:optional true} [:sequential :map]]- [:transforms {:optional true} [:map-of ::state-map-key :map]]+ [:map {:closed true}+ [:queries {:optional true} [:map-of ::state-map-key ::query]]+ [:charts {:optional true} [:map-of ::state-map-key ::chart]]+ [:chart-configs {:optional true} [:map-of ::state-map-key ::chart-config]]+ [:todos {:optional true} [:sequential ::todo]]+ [:transforms {:optional true} [:map-of ::state-map-key ::transform]][:link-registry {:optional true} [:map-of ::state-map-key :string]]])(defn normalize-state@@ -110,22 +219,10 @@(mr/def ::ai-usage-log"A AiUsageLog as selected from the app DB: every column of `:ai_usage_log`."- [:map {:closed true}- [:id ms/PositiveInt]- [:created_at ms/TemporalInstant]- [:source [:or :keyword :string]]- [:model [:or :keyword :string]]- [:prompt_tokens :int]- [:completion_tokens :int]- [:total_tokens :int]- [:user_id [:maybe ::lib.schema.id/user]]- [:tenant_id [:maybe ms/PositiveInt]]- [:conversation_id [:maybe :string]]- [:profile_id [:maybe :string]]- [:request_id [:maybe :string]]- [:ai_proxied [:maybe :boolean]]- [:cache_creation_tokens [:maybe :int]]- [:cache_read_tokens [:maybe :int]]])+ [:merge+ ::ai-usage-log.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::ai-usage-log.update"What an update (or insert) of a AiUsageLog accepts: every column of `:ai_usage_log` except `id`, all optional."@@ -147,15 +244,10 @@(mr/def ::metabot"A Metabot as selected from the app DB: every column of `:metabot`."- [:map {:closed true}- [:id ms/PositiveInt]- [:name :string]- [:description [:maybe :string]]- [:entity_id :string]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]- [:use_verified_content [:maybe :boolean]]- [:collection_id [:maybe ::lib.schema.id/collection]]])+ [:merge+ ::metabot.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::metabot.update"What an update (or insert) of a Metabot accepts: every column of `:metabot` except `id`, all optional."@@ -170,20 +262,10 @@(mr/def ::metabot-conversation"A MetabotConversation as selected from the app DB: every column of `:metabot_conversation`."- [:map {:closed true}- [:id :string]- [:created_at ms/TemporalInstant]- [:user_id ::lib.schema.id/user]- [:title [:maybe :string]]- [:ip_address [:maybe :string]]- [:slack_team_id [:maybe :string]]- [:slack_channel_id [:maybe :string]]- [:slack_thread_ts [:maybe :string]]- [:embedding_hostname [:maybe :string]]- [:embedding_path [:maybe :string]]- [:user_agent [:maybe :string]]- [:sanitized_user_agent [:maybe :string]]- [:forked_from_conversation_id [:maybe :string]]])+ [:merge+ ::metabot-conversation.update+ [:map {:closed true}+ [:id :string]]])(mr/def ::metabot-conversation.update"What an update (or insert) of a MetabotConversation accepts: every column of `:metabot_conversation` except `id`, all optional."@@ -203,15 +285,10 @@(mr/def ::metabot-feedback"A MetabotFeedback as selected from the app DB: every column of `:metabot_feedback`."- [:map {:closed true}- [:message_id ms/PositiveInt]- [:positive :boolean]- [:issue_type [:maybe [:or :keyword :string]]]- [:freeform_feedback [:maybe :string]]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]- [:id ms/PositiveInt]- [:user_id ::lib.schema.id/user]])+ [:merge+ ::metabot-feedback.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::metabot-feedback.update"What an update (or insert) of a MetabotFeedback accepts: every column of `:metabot_feedback` except `id`, all optional."@@ -226,40 +303,29 @@(mr/def ::metabot-message.data-part"One entry of the `:data` column of a MetabotMessage, decoded."- :map)+ ::schema.v2/ui-message-part)(mr/def ::metabot-message.usage"The `:usage` column of a MetabotMessage, decoded."- :map)+ (ms/string-keyed-map [:map {:closed true}+ [:prompt :int]+ [:completion :int]]))++(defn normalize-usage+ "Normalize the model keys of a MetabotMessage `:usage` value to strings according to [[::metabot-message.usage]]."+ [usage]+ (mc/decode ::metabot-message.usage usage (mtx/transformer {:name :normalize})))(mr/def ::metabot-message.state"The `:state` column of a MetabotMessage, decoded."- :map)+ ::state)(mr/def ::metabot-message"A MetabotMessage as selected from the app DB: every column of `:metabot_message`."- [:map {:closed true}- [:id ms/PositiveInt]- [:created_at ms/TemporalInstant]- [:profile_id :string]- [:role [:or :keyword :string]]- [:data [:sequential ::metabot-message.data-part]]- [:usage [:maybe ::metabot-message.usage]]- [:total_tokens :int]- [:conversation_id :string]- [:slack_msg_id [:maybe :string]]- [:channel_id [:maybe :string]]- [:deleted_at [:maybe ms/TemporalInstant]]- [:deleted_by_user_id [:maybe ::lib.schema.id/user]]- [:user_id [:maybe ::lib.schema.id/user]]- [:ai_proxied [:maybe :boolean]]- [:external_id [:maybe :string]]- [:finished [:maybe :boolean]]- [:error [:maybe :string]]- [:data_version :int]- [:state [:maybe ::metabot-message.state]]- [:forked_from_message_id [:maybe ms/PositiveInt]]- [:context_tokens [:maybe :int]]])+ [:merge+ ::metabot-message.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::metabot-message.update"What an update (or insert) of a MetabotMessage accepts: every column of `:metabot_message` except `id`, all optional."@@ -287,15 +353,10 @@(mr/def ::metabot-prompt"A MetabotPrompt as selected from the app DB: every column of `:metabot_prompt`."- [:map {:closed true}- [:id ms/PositiveInt]- [:model [:or :keyword :string]]- [:card_id ::lib.schema.id/card]- [:entity_id :string]- [:prompt :string]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]- [:metabot_id ms/PositiveInt]])+ [:merge+ ::metabot-prompt.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::metabot-prompt.update"What an update (or insert) of a MetabotPrompt accepts: every column of `:metabot_prompt` except `id`, all optional."@@ -310,15 +371,10 @@(mr/def ::metabot-source-feedback"A MetabotSourceFeedback as selected from the app DB: every column of `:metabot_source_feedback`."- [:map {:closed true}- [:id ms/PositiveInt]- [:message_id ms/PositiveInt]- [:user_id ::lib.schema.id/user]- [:source_id ms/PositiveInt]- [:source_type [:or :keyword :string]]- [:positive :boolean]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]])+ [:merge+ ::metabot-source-feedback.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::metabot-source-feedback.update"What an update (or insert) of a MetabotSourceFeedback accepts: every column of `:metabot_source_feedback` except `id`, all optional."@@ -333,11 +389,10 @@(mr/def ::metabot-used-table"A MetabotUsedTable as selected from the app DB: every column of `:metabot_used_table`."- [:map {:closed true}- [:id ms/PositiveInt]- [:message_id ms/PositiveInt]- [:table_id ::lib.schema.id/table]- [:created_at ms/TemporalInstant]])+ [:merge+ ::metabot-used-table.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::metabot-used-table.update"What an update (or insert) of a MetabotUsedTable accepts: every column of `:metabot_used_table` except `id`, all optional."diff --git a/src/metabase/metabot/schema/v2.clj b/src/metabase/metabot/schema/v2.cljindex 4c580e6f28d..b2957c7e930 100644--- a/src/metabase/metabot/schema/v2.clj+++ b/src/metabase/metabot/schema/v2.clj@@ -60,13 +60,133 @@[part](subs (:type part) (count "tool-")))+(mr/def ::anthropic-provider-metadata+ "Anthropic-specific data carried on a reasoning part: a redacted-thinking block's opaque+ payload, or a signed-thinking block's signature."+ [:map {:closed true}+ [:redactedData {:optional true} [:maybe :string]]+ [:signature {:optional true} [:maybe :string]]])++(mr/def ::openai-provider-metadata+ "OpenAI-specific data carried on a reasoning part, needed to replay it across tool-call+ round-trips despite `store:false`."+ [:map {:closed true}+ [:encryptedContent {:optional true} [:maybe :string]]+ [:itemId {:optional true} [:maybe :string]]])++(mr/def ::google-provider-metadata+ "Google-specific data carried on a tool-input part: the thought signature Gemini 3.x requires+ when a functionCall is replayed in the current turn."+ [:map {:closed true}+ [:thoughtSignature {:optional true} [:maybe :string]]])+(mr/def ::provider-metadata- [:map-of :keyword [:map-of :keyword :any]])+ [:map {:closed true}+ [:anthropic {:optional true} [:maybe ::anthropic-provider-metadata]]+ [:openai {:optional true} [:maybe ::openai-provider-metadata]]+ [:google {:optional true} [:maybe ::google-provider-metadata]]])(mr/def ::never"`z.never().optional()`: the key must be absent. (zod also tolerates a present `undefined`value, which JSON cannot represent.)"- [:not :any])+ [:fn {:error/message "must be absent"} (fn [_] false)])++(mr/def ::tool-payload+ "A metabot tool's `:structured-output`/`:structured_output`, `:resources`, or `:data-parts` data: a+ closed bag of the field names metabot tools actually set."+ [:map {:closed true}+ [:result-type {:optional true} [:maybe [:or :string :keyword]]]+ [:type {:optional true} [:maybe [:or :string :keyword]]]+ [:list-type {:optional true} [:maybe [:or :string :keyword]]]+ [:message {:optional true} [:maybe :string]]+ [:path {:optional true} [:maybe :string]]+ [:question {:optional true} [:maybe :string]]+ [:options {:optional true} [:maybe [:sequential :string]]]+ [:data {:optional true} [:maybe [:or [:ref ::tool-payload] [:sequential [:ref ::tool-payload]]]]]+ [:total_count {:optional true} [:maybe :int]]+ [:weak_match {:optional true} [:maybe :boolean]]+ [:tables {:optional true} [:maybe [:sequential [:ref ::tool-payload]]]]+ [:models {:optional true} [:maybe [:sequential [:ref ::tool-payload]]]]+ [:metrics {:optional true} [:maybe [:sequential [:ref ::tool-payload]]]]+ [:errors {:optional true} [:maybe [:sequential [:ref ::tool-payload]]]]+ [:results {:optional true} [:maybe [:sequential [:ref ::tool-payload]]]]+ [:id {:optional true} [:maybe [:or :int :string]]]+ [:name {:optional true} [:maybe :string]]+ [:description {:optional true} [:maybe :string]]+ [:content {:optional true} [:maybe :string]]+ [:document {:optional true} [:maybe :string]]+ [:items {:optional true} [:maybe [:sequential [:ref ::tool-payload]]]]+ [:total {:optional true} [:maybe :int]]+ [:page {:optional true} [:maybe :int]]+ [:pages {:optional true} [:maybe :int]]+ [:card-id {:optional true} [:maybe :int]]+ [:card_id {:optional true} [:maybe :int]]+ [:collection-id {:optional true} [:maybe :int]]+ [:collection_id {:optional true} [:maybe :int]]+ [:destination {:optional true} [:maybe [:or :string [:ref ::tool-payload]]]]+ [:todos {:optional true} [:maybe [:sequential :metabase.metabot.schema/todo]]]+ [:todo_count {:optional true} [:maybe :int]]+ [:events {:optional true} [:maybe [:sequential [:ref ::tool-payload]]]]+ [:timestamp {:optional true} [:maybe :string]]+ [:time_matters {:optional true} [:maybe :boolean]]+ [:timezone {:optional true} [:maybe :string]]+ [:status {:optional true} [:maybe :string]]+ [:priority {:optional true} [:maybe :string]]+ [:verified {:optional true} [:maybe :boolean]]+ [:next-page-uri {:optional true} [:maybe :string]]+ [:tabs {:optional true} [:maybe [:sequential [:ref ::tool-payload]]]]+ [:approved {:optional true} [:maybe :boolean]]+ [:success {:optional true} [:maybe :boolean]]+ [:bad_transforms {:optional true} [:maybe [:sequential [:ref ::tool-payload]]]]+ [:bad_questions {:optional true} [:maybe [:sequential [:ref ::tool-payload]]]]+ [:query {:optional true} [:maybe :metabase.metabot.schema/query]]+ [:query-id {:optional true} [:maybe :string]]+ [:query_id {:optional true} [:maybe :string]]+ [:query-content {:optional true} [:maybe :string]]+ [:query-json {:optional true} [:maybe :string]]+ [:result-columns {:optional true} [:maybe [:sequential :string]]]+ [:database {:optional true} [:maybe :int]]+ [:database_id {:optional true} [:maybe :int]]+ [:sql_engine {:optional true} [:maybe :string]]+ [:chart-type {:optional true} [:maybe [:or :string :keyword]]]+ [:chart_type {:optional true} [:maybe [:or :string :keyword]]]+ [:chart-id {:optional true} [:maybe :string]]+ [:display {:optional true} [:maybe [:or :string :keyword]]]+ [:tool {:optional true} [:maybe :string]]+ [:dataset_query {:optional true} [:maybe :metabase.metabot.schema/query]]+ [:transform {:optional true} [:maybe :metabase.metabot.schema/transform]]+ [:target {:optional true} [:maybe [:ref ::tool-payload]]]+ [:source {:optional true} [:maybe [:ref ::tool-payload]]]+ [:schema {:optional true} [:maybe :string]]+ [:url {:optional true} [:maybe :string]]+ [:title {:optional true} [:maybe :string]]+ [:entity_id {:optional true} [:maybe :int]]+ [:link {:optional true} [:maybe :string]]])++(mr/def ::structured-output+ "The `persisted-structured-output-keys` subset of a tool's `:structured-output`/+ `:structured_output`, as stored on a v2 tool part's `:output`+ (see `metabase.metabot.persistence/tool-result->storable-output`)."+ [:map {:closed true}+ [:query-id {:optional true} [:maybe :string]]+ [:query-content {:optional true} [:maybe :string]]+ [:query {:optional true} [:maybe :metabase.metabot.schema/query]]+ [:database {:optional true} [:maybe :int]]+ [:chart-type {:optional true} [:maybe [:or :string :keyword]]]])++(mr/def ::stored-tool-output+ "A v2 tool part's `:output`: the trimmed value+ `metabase.metabot.persistence/tool-result->storable-output` stores, either a tool's bare+ scalar result or the `:output`/`:structured_output` map trimmed from it."+ [:or+ :string+ :keyword+ number?+ :boolean+ :nil+ [:map {:closed true}+ [:output {:optional true} [:maybe :string]]+ [:structured_output {:optional true} [:maybe ::structured-output]]]])(mr/def ::ui-message-chunk[:multi {:dispatch (fn [chunk]@@ -194,187 +314,192 @@[:type [:= "message-metadata"]][:messageMetadata {:optional true} :any]]]])+(mr/def ::tool-io+ "A tool call's `:input`/`:rawInput`/`:output`: an arbitrary JSON value whose shape the calling tool (or the LLM+ provider) owns, not us."+ [:schema {::mr/deliberately-open true, :description "arbitrary tool call arguments or result"} :any])+(mr/def ::tool-ui-part[:multi {:dispatch :state}- ["input-streaming" [:map+ ["input-streaming" [:map {:closed true}[:type [:fn tool-type?]][:toolCallId :string][:state [:= "input-streaming"]][:providerExecuted {:optional true} :boolean]- [:input {:optional true} :any]+ [:input {:optional true} ::tool-io][:output {:optional true} ::never][:errorText {:optional true} ::never][:approval {:optional true} ::never]]]- ["input-available" [:map+ ["input-available" [:map {:closed true}[:type [:fn tool-type?]][:toolCallId :string][:state [:= "input-available"]][:providerExecuted {:optional true} :boolean]- [:input {:optional true} :any]+ [:input {:optional true} ::tool-io][:output {:optional true} ::never][:errorText {:optional true} ::never][:callProviderMetadata {:optional true} ::provider-metadata][:approval {:optional true} ::never]]]- ["approval-requested" [:map+ ["approval-requested" [:map {:closed true}[:type [:fn tool-type?]][:toolCallId :string][:state [:= "approval-requested"]]- [:input {:optional true} :any]+ [:input {:optional true} ::tool-io][:providerExecuted {:optional true} :boolean][:output {:optional true} ::never][:errorText {:optional true} ::never][:callProviderMetadata {:optional true} ::provider-metadata]- [:approval [:map+ [:approval [:map {:closed true}[:id :string][:approved {:optional true} ::never][:reason {:optional true} ::never]]]]]- ["approval-responded" [:map+ ["approval-responded" [:map {:closed true}[:type [:fn tool-type?]][:toolCallId :string][:state [:= "approval-responded"]]- [:input {:optional true} :any]+ [:input {:optional true} ::tool-io][:providerExecuted {:optional true} :boolean][:output {:optional true} ::never][:errorText {:optional true} ::never][:callProviderMetadata {:optional true} ::provider-metadata]- [:approval [:map+ [:approval [:map {:closed true}[:id :string][:approved :boolean][:reason {:optional true} :string]]]]]- ["output-available" [:map+ ["output-available" [:map {:closed true}[:type [:fn tool-type?]][:toolCallId :string][:state [:= "output-available"]][:providerExecuted {:optional true} :boolean]- [:input {:optional true} :any]- [:output {:optional true} :any]+ [:input {:optional true} ::tool-io]+ [:output {:optional true} ::stored-tool-output][:errorText {:optional true} ::never][:callProviderMetadata {:optional true} ::provider-metadata][:preliminary {:optional true} :boolean]- [:approval {:optional true} [:map+ [:approval {:optional true} [:map {:closed true}[:id :string][:approved [:= true]][:reason {:optional true} :string]]]]]- ["output-error" [:map+ ["output-error" [:map {:closed true}[:type [:fn tool-type?]][:toolCallId :string][:state [:= "output-error"]][:providerExecuted {:optional true} :boolean]- [:input {:optional true} :any]- [:rawInput {:optional true} :any]+ [:input {:optional true} ::tool-io]+ [:rawInput {:optional true} ::tool-io][:output {:optional true} ::never][:errorText :string][:callProviderMetadata {:optional true} ::provider-metadata]- [:approval {:optional true} [:map+ [:approval {:optional true} [:map {:closed true}[:id :string][:approved [:= true]][:reason {:optional true} :string]]]]]- ["output-denied" [:map+ ["output-denied" [:map {:closed true}[:type [:fn tool-type?]][:toolCallId :string][:state [:= "output-denied"]][:providerExecuted {:optional true} :boolean]- [:input {:optional true} :any]+ [:input {:optional true} ::tool-io][:output {:optional true} ::never][:errorText {:optional true} ::never][:callProviderMetadata {:optional true} ::provider-metadata]- [:approval [:map+ [:approval [:map {:closed true}[:id :string][:approved [:= false]][:reason {:optional true} :string]]]]]])(mr/def ::dynamic-tool-ui-part[:multi {:dispatch :state}- ["input-streaming" [:map+ ["input-streaming" [:map {:closed true}[:type [:= "dynamic-tool"]][:toolName :string][:toolCallId :string][:state [:= "input-streaming"]]- [:input {:optional true} :any]+ [:input {:optional true} ::tool-io][:providerExecuted {:optional true} :boolean][:output {:optional true} ::never][:errorText {:optional true} ::never][:approval {:optional true} ::never]]]- ["input-available" [:map+ ["input-available" [:map {:closed true}[:type [:= "dynamic-tool"]][:toolName :string][:toolCallId :string][:state [:= "input-available"]]- [:input {:optional true} :any]+ [:input {:optional true} ::tool-io][:providerExecuted {:optional true} :boolean][:output {:optional true} ::never][:errorText {:optional true} ::never][:callProviderMetadata {:optional true} ::provider-metadata][:approval {:optional true} ::never]]]- ["approval-requested" [:map+ ["approval-requested" [:map {:closed true}[:type [:= "dynamic-tool"]][:toolName :string][:toolCallId :string][:state [:= "approval-requested"]]- [:input {:optional true} :any]+ [:input {:optional true} ::tool-io][:providerExecuted {:optional true} :boolean][:output {:optional true} ::never][:errorText {:optional true} ::never][:callProviderMetadata {:optional true} ::provider-metadata]- [:approval [:map+ [:approval [:map {:closed true}[:id :string][:approved {:optional true} ::never][:reason {:optional true} ::never]]]]]- ["approval-responded" [:map+ ["approval-responded" [:map {:closed true}[:type [:= "dynamic-tool"]][:toolName :string][:toolCallId :string][:state [:= "approval-responded"]]- [:input {:optional true} :any]+ [:input {:optional true} ::tool-io][:providerExecuted {:optional true} :boolean][:output {:optional true} ::never][:errorText {:optional true} ::never][:callProviderMetadata {:optional true} ::provider-metadata]- [:approval [:map+ [:approval [:map {:closed true}[:id :string][:approved :boolean][:reason {:optional true} :string]]]]]- ["output-available" [:map+ ["output-available" [:map {:closed true}[:type [:= "dynamic-tool"]][:toolName :string][:toolCallId :string][:state [:= "output-available"]]- [:input {:optional true} :any]+ [:input {:optional true} ::tool-io][:providerExecuted {:optional true} :boolean]- [:output {:optional true} :any]+ [:output {:optional true} ::stored-tool-output][:errorText {:optional true} ::never][:callProviderMetadata {:optional true} ::provider-metadata][:preliminary {:optional true} :boolean]- [:approval {:optional true} [:map+ [:approval {:optional true} [:map {:closed true}[:id :string][:approved [:= true]][:reason {:optional true} :string]]]]]- ["output-error" [:map+ ["output-error" [:map {:closed true}[:type [:= "dynamic-tool"]][:toolName :string][:toolCallId :string][:state [:= "output-error"]]- [:input {:optional true} :any]- [:rawInput {:optional true} :any]+ [:input {:optional true} ::tool-io]+ [:rawInput {:optional true} ::tool-io][:providerExecuted {:optional true} :boolean][:output {:optional true} ::never][:errorText :string][:callProviderMetadata {:optional true} ::provider-metadata]- [:approval {:optional true} [:map+ [:approval {:optional true} [:map {:closed true}[:id :string][:approved [:= true]][:reason {:optional true} :string]]]]]- ["output-denied" [:map+ ["output-denied" [:map {:closed true}[:type [:= "dynamic-tool"]][:toolName :string][:toolCallId :string][:state [:= "output-denied"]]- [:input {:optional true} :any]+ [:input {:optional true} ::tool-io][:providerExecuted {:optional true} :boolean][:output {:optional true} ::never][:errorText {:optional true} ::never][:callProviderMetadata {:optional true} ::provider-metadata]- [:approval [:map+ [:approval [:map {:closed true}[:id :string][:approved [:= false]][:reason {:optional true} :string]]]]]])@@ -387,43 +512,43 @@(tool-type? t) ::tool(data-type? t) ::data:else t)))}- ["text" [:map+ ["text" [:map {:closed true}[:type [:= "text"]][:text :string][:state {:optional true} [:enum "streaming" "done"]][:providerMetadata {:optional true} ::provider-metadata]]]- ["reasoning" [:map+ ["reasoning" [:map {:closed true}[:type [:= "reasoning"]][:text :string][:state {:optional true} [:enum "streaming" "done"]][:providerMetadata {:optional true} ::provider-metadata]]]- ["source-url" [:map+ ["source-url" [:map {:closed true}[:type [:= "source-url"]][:sourceId :string][:url :string][:title {:optional true} :string][:providerMetadata {:optional true} ::provider-metadata]]]- ["source-document" [:map+ ["source-document" [:map {:closed true}[:type [:= "source-document"]][:sourceId :string][:mediaType :string][:title :string][:filename {:optional true} :string][:providerMetadata {:optional true} ::provider-metadata]]]- ["file" [:map+ ["file" [:map {:closed true}[:type [:= "file"]][:mediaType :string][:filename {:optional true} :string][:url :string][:providerMetadata {:optional true} ::provider-metadata]]]- ["step-start" [:map+ ["step-start" [:map {:closed true}[:type [:= "step-start"]]]]["dynamic-tool" ::dynamic-tool-ui-part][::tool ::tool-ui-part]- [::data [:map+ [::data [:map {:closed true}[:type [:fn data-type?]][:id {:optional true} :string]- [:data {:optional true} :any]]]])+ [:data {:optional true} [:maybe ::tool-io]]]]])(mr/def ::ui-message[:mapdiff --git a/src/metabase/metabot/self/core.clj b/src/metabase/metabot/self/core.cljindex 30371e49dfa..97ffa73cabd 100644--- a/src/metabase/metabot/self/core.clj+++ b/src/metabase/metabot/self/core.clj@@ -9,14 +9,17 @@[malli.transform :as mtx][metabase.ai-tracing.core :as ait][metabase.llm.settings :as llm]+ [metabase.metabot.schema :as metabot.schema][metabase.metabot.schema.v2 :as schema.v2][metabase.premium-features.core :as premium-features]+ [metabase.request.schema :as request.schema][metabase.settings.core :as setting][metabase.util :as u][metabase.util.i18n :refer [tru]][metabase.util.json :as json][metabase.util.log :as log][metabase.util.malli.registry :as mr]+ [metabase.util.malli.schema :as ms][metabase.util.o11y :refer [with-span]])(:import(java.io BufferedReader Closeable InputStream)@@ -31,13 +34,174 @@;; "required" → {:type "any"} for Claude, system-message placement, tool wire;; format) happens inside each adapter, but the **input contract is identical**.+(def ^:private MalliSchema+ "A malli schema, in schema-form or as a compiled instance."+ [:and :any [:fn {:error/message "a malli schema"} mc/schema]])++(def ^:private AnthropicProviderMetadata+ "Anthropic-specific data carried on a reasoning part: a redacted-thinking block's opaque+ payload, or a signed-thinking block's signature."+ [:map {:closed true}+ [:redactedData {:optional true} [:maybe :string]]+ [:signature {:optional true} [:maybe :string]]])++(def ^:private OpenAIProviderMetadata+ "OpenAI-specific data carried on a reasoning part, needed to replay it across tool-call+ round-trips despite `store:false`."+ [:map {:closed true}+ [:encryptedContent {:optional true} [:maybe :string]]+ [:itemId {:optional true} [:maybe :string]]])++(def ^:private MistralProviderMetadata+ "Mistral-specific data carried on a reasoning part: a think chunk's captured signature."+ [:map {:closed true}+ [:signature {:optional true} [:maybe :string]]])++(def ^:private GoogleProviderMetadata+ "Google-specific data carried on a tool-input part: the thought signature Gemini 3.x requires+ when a functionCall is replayed in the current turn."+ [:map {:closed true}+ [:thoughtSignature {:optional true} [:maybe :string]]])++(def ^:private ProviderMetadata+ "Vendor-specific data carried verbatim on a reasoning/tool-input part, namespaced by provider."+ [:map {:closed true}+ [:anthropic {:optional true} [:maybe AnthropicProviderMetadata]]+ [:openai {:optional true} [:maybe OpenAIProviderMetadata]]+ [:google {:optional true} [:maybe GoogleProviderMetadata]]+ [:mistral {:optional true} [:maybe MistralProviderMetadata]]])+(def ToolEntry"A tool definition map with :tool-name, :doc, :schema, :fn, and optionally :decode/:prompt."- [:map+ [:map {:closed true}[:tool-name :string][:doc {:optional true} [:maybe :string]]- [:schema :any]- [:fn [:fn fn?]]])+ [:schema MalliSchema]+ [:fn [:fn fn?]]+ [:decode {:optional true} [:maybe [:fn fn?]]]+ [:prompt {:optional true} [:maybe :string]]+ [:title-fn {:optional true} [:maybe [:fn fn?]]]+ [:system-instructions {:optional true} [:maybe :string]]+ [:capabilities {:optional true} [:maybe [:set :keyword]]]+ [:scope {:optional true} [:maybe :string]]])++(def ^:private RawStructuredOutput+ "A tool's own `:structured-output`/`:structured_output` map before+ `metabase.metabot.persistence/tool-result->storable-output` trims it down to+ `metabase.metabot.schema.v2/structured-output` for persistence."+ [:or ::metabot.schema/transform ::schema.v2/tool-payload [:sequential ::schema.v2/tool-payload]])++(def ^:private DataPart+ "One entry of a tool's `:data-parts`: `metabase.metabot.agent.streaming`'s `{:type :data, ...}`+ constructors."+ [:map {:closed true}+ [:type [:= :data]]+ [:data-type :string]+ [:data {:optional true} [:maybe ::schema.v2/tool-payload]]])++(def ^:private ToolResult+ "The raw return value of a tool's `:fn`, before it is trimmed for persistence or forwarded to+ a provider (see [[collect-tool-result]])."+ [:or+ :string+ :keyword+ number?+ :boolean+ :nil+ [:map {:closed true}+ [:output {:optional true} [:maybe :string]]+ [:structured-output {:optional true} [:maybe RawStructuredOutput]]+ [:structured_output {:optional true} [:maybe RawStructuredOutput]]+ [:terminal-error? {:optional true} :boolean]+ [:data-parts {:optional true} [:sequential DataPart]]+ [:resources {:optional true} [:sequential ::schema.v2/tool-payload]]+ [:instructions {:optional true} [:maybe :string]]+ [:status-code {:optional true} [:maybe :int]]+ [:error {:optional true} [:maybe [:map {:closed true}+ [:message {:optional true} [:maybe :string]]+ [:type {:optional true} [:maybe :string]]]]]]])++(def ^:private ToolCallArguments+ "A tool call's arguments as the LLM wrote them against the tool's own schema, keyed by that tool's argument names:+ string keys off the wire, keyword keys when built in Clojure."+ [:map-of {::mr/deliberately-open true, :description "tool call arguments"}+ [:or :string :keyword] ::request.schema/json-value])++(def ^:private AISDKPart+ "One element of the `:input` sequence passed to a provider adapter: an AISDK part keyed by+ `:type` (`:text`, `:reasoning`, `:tool-input`, `:tool-output`), or a plain role message keyed+ by `:role` instead."+ [:map {:closed true}+ [:type {:optional true} [:maybe :keyword]]+ [:role {:optional true} [:maybe (ms/enum-keywords-and-strings :user :system :assistant :tool)]]+ [:id {:optional true} [:maybe :string]]+ [:text {:optional true} [:maybe :string]]+ [:content {:optional true} [:maybe :string]]+ [:function {:optional true} [:maybe :string]]+ [:arguments {:optional true} [:maybe ToolCallArguments]]+ [:result {:optional true} [:maybe ToolResult]]+ [:error {:optional true} [:maybe [:map {:closed true}+ [:message {:optional true} [:maybe :string]]+ [:type {:optional true} [:maybe :string]]]]]+ [:provider-metadata {:optional true} [:maybe ProviderMetadata]]])++(def ^:private ApiKeyCredentials+ "The `{:api-key ... :base-url ...}` connection shape shared by most providers."+ [:map {:closed true}+ [:api-key {:optional true} [:maybe :string]]+ [:base-url {:optional true} [:maybe :string]]+ [:model-reasoning {:optional true} [:maybe [:or :boolean :string]]]])++(def ^:private BedrockCredentials+ [:map {:closed true}+ [:access-key-id {:optional true} [:maybe :string]]+ [:secret-access-key {:optional true} [:maybe :string]]+ [:session-token {:optional true} [:maybe :string]]+ [:region {:optional true} [:maybe :string]]])++(def ^:private GoogleCredentials+ [:map {:closed true}+ [:service-account-key {:optional true} [:maybe :string]]+ [:oauth-access-token {:optional true} [:maybe :string]]+ [:project-id {:optional true} [:maybe :string]]+ [:location {:optional true} [:maybe :string]]+ [:base-url {:optional true} [:maybe :string]]])++(def ^:private LLMCredentials+ [:or ApiKeyCredentials BedrockCredentials GoogleCredentials])++(def ^:private ReasoningConfig+ "A dialect-shaped reasoning/thinking directive, sent verbatim to the provider."+ [:map {:closed true}+ [:type :string]+ [:display {:optional true} [:maybe :string]]])++(def ^:private JSONSchemaLeaf+ "A leaf JSON Schema node: no `:properties` of its own, one further leaf level of `:items` for+ an array-typed leaf."+ [:map {:closed true}+ [:type {:optional true} [:maybe :string]]+ [:description {:optional true} [:maybe :string]]+ [:items {:optional true} [:map {:closed true}+ [:type {:optional true} [:maybe :string]]+ [:description {:optional true} [:maybe :string]]]]+ [:minimum {:optional true} number?]+ [:maximum {:optional true} number?]])++(def ^:private JSONSchemaProperties+ "The `:properties` of a JSON Schema node, keyed by the field names the caller's structured-output schema declares:+ string keys off the wire, keyword keys when built in Clojure."+ [:map-of {::mr/deliberately-open true, :description "JSON Schema properties"}+ [:or :string :keyword] JSONSchemaLeaf])++(def ^:private JSONSchemaNode+ "A JSON Schema node, sent verbatim to an LLM provider as the structured-output schema.+ `:properties` keys are the field names the schema itself declares, not ours to enumerate."+ [:map {:closed true}+ [:type {:optional true} [:maybe :string]]+ [:properties {:optional true} JSONSchemaProperties]+ [:required {:optional true} [:vector :string]]+ [:additionalProperties {:optional true} :boolean]])(def LLMRequestOpts"Canonical schema for the opts map passed to every LLM provider adapter.@@ -69,19 +233,19 @@ignore it:prompt-cache-key - prompt-cache affinity hint (the conversation id); adapters whoseprovider caches opt-in per key forward it (Mistral), others ignore it"- [:map+ [:map {:closed true}[:model {:optional true} :string][:system {:optional true} [:maybe :string]]- [:input {:optional true} [:sequential :map]]+ [:input {:optional true} [:sequential AISDKPart]][:tools {:optional true} [:maybe [:sequential ToolEntry]]][:tool_choice {:optional true} [:maybe [:enum "auto" "required"]]][:temperature {:optional true} [:maybe number?]][:max-tokens {:optional true} [:maybe :int]]- [:schema {:optional true} :any]- [:credentials {:optional true} [:maybe :map]]+ [:schema {:optional true} [:maybe JSONSchemaNode]]+ [:credentials {:optional true} [:maybe LLMCredentials]][:ai-proxy? {:optional true} [:maybe :boolean]][:reasoning? {:optional true} [:maybe :boolean]]- [:reasoning-config {:optional true} [:maybe :map]]+ [:reasoning-config {:optional true} [:maybe ReasoningConfig]][:fast? {:optional true} [:maybe :boolean]][:prompt-cache-key {:optional true} [:maybe :string]]])@@ -329,7 +493,7 @@(get tools (:function part)))](if title-fn(let [title (try- (title-fn (cond-> (coerce-stringified-json (:arguments part))+ (title-fn (cond-> (walk/keywordize-keys (coerce-stringified-json (:arguments part)))decode decode))(catch Throwable e(log/debug e "tool title-fn failed" {:tool (:function part)})@@ -776,7 +940,7 @@(= (:type chunk) :tool-output-available) (assoc ::duration-ms duration-ms))))results (try(let [{:keys [arguments]} (into {} (aisdk-xf) chunks)- arguments (or (coerce-stringified-json arguments) {})+ arguments (walk/keywordize-keys (or (coerce-stringified-json arguments) {}))arguments (coerce-stringified-scalars tool arguments)decode (tool-decode-fn tool)arguments (cond-> arguments decode decode)diff --git a/src/metabase/metabot/self/google/raw_predict.clj b/src/metabase/metabot/self/google/raw_predict.cljindex d769de7b57c..f873f8488b8 100644--- a/src/metabase/metabot/self/google/raw_predict.clj+++ b/src/metabase/metabot/self/google/raw_predict.clj@@ -30,7 +30,7 @@(mu/defn request-body"Build the streamRawPredict request body for an LLM request on `model-id`."- [model-id opts :- core/LLMRequestOpts]+ [model-id :- [:maybe :string] opts :- core/LLMRequestOpts](-> (claude/claude-request-body (assoc opts :model (direct-api-model-id model-id)))(dissoc :model)(assoc :anthropic_version anthropic-version)))diff --git a/src/metabase/metabot/self/openai/chat_completions.clj b/src/metabase/metabot/self/openai/chat_completions.cljindex db682d54017..129484496fc 100644--- a/src/metabase/metabot/self/openai/chat_completions.clj+++ b/src/metabase/metabot/self/openai/chat_completions.clj@@ -328,15 +328,20 @@;;; Request body+(def ^:private CCOpts+ "Dialect hooks for [[request-body]] that are not request options."+ [:maybe [:map {:closed true}+ [:reasoning-part->message {:optional true} [:maybe [:fn fn?]]]]])+(mu/defn request-body"Build the Chat Completions request body for an LLM request.The optional `cc-opts` map holds dialect hooks that are not request options —today only `:reasoning-part->message`, threaded to [[parts->cc-messages]]. Afn-valued hook stays out of the traced and logged `LLMRequestOpts` on purpose."- ([opts] (request-body opts nil))+ ([opts :- core/LLMRequestOpts] (request-body opts nil))([{:keys [model system input tools temperature max-tokens tool_choice schema]} :- core/LLMRequestOpts- cc-opts]+ cc-opts :- CCOpts](let [messages (cond-> (parts->cc-messages input cc-opts)system (as-> msgs (into [{:role "system" :content system}] msgs)))all-tools (or (when schemadiff --git a/src/metabase/metabot/tools/autogen_dashboard.clj b/src/metabase/metabot/tools/autogen_dashboard.cljindex cd8cd1f5c97..d24e04c5a18 100644--- a/src/metabase/metabot/tools/autogen_dashboard.clj+++ b/src/metabase/metabot/tools/autogen_dashboard.clj@@ -119,7 +119,7 @@(def ^:private autogen-dashboard-schema[:map {:closed true}- [:source [:map+ [:source [:map {:closed true}[:table_id {:optional true} [:maybe :int]][:model_id {:optional true} [:maybe :int]][:metric_id {:optional true} [:maybe :int]]diff --git a/src/metabase/metabot/tools/construct.clj b/src/metabase/metabot/tools/construct.cljindex 094ee7c30db..7d95a6bc7b1 100644--- a/src/metabase/metabot/tools/construct.clj+++ b/src/metabase/metabot/tools/construct.clj@@ -90,6 +90,13 @@"joins" {:type "array" :items {:type "object"}}"expressions" {:type "object"}}}}}})+(def LLMExternalQuery+ "An MBQL 5 external query as the LLM wrote it, before [[execute-representations-query]] repairs and validates it: its+ keys are the LLM's until then."+ [:map {:closed false+ ::mr/deliberately-open true+ :description "An LLM-authored MBQL 5 query, structurally unvalidated here; real validation happens at the entry-point boundaries."}])+(def ^:private construct-notebook-query-args-schema"Args schema for `construct_notebook_query`.@@ -108,7 +115,7 @@[:reasoning {:optional true} :string];; Validation stays a fully open, property-less `:map` (the repair layer fixes LLM shortcuts); the;; `:json-schema` override only changes what the LLM sees. See [[construct-notebook-query-json-schema]].- [:query [:map {:json-schema construct-notebook-query-json-schema}]]+ [:query (mu/with LLMExternalQuery {:json-schema construct-notebook-query-json-schema})][:visualization {:optional true} construct-visualization-schema][:title :string][:description :string]])diff --git a/src/metabase/metabot/tools/slackbot_query.clj b/src/metabase/metabot/tools/slackbot_query.cljindex 18492720215..a24254cafc0 100644--- a/src/metabase/metabot/tools/slackbot_query.clj+++ b/src/metabase/metabot/tools/slackbot_query.clj@@ -25,7 +25,7 @@`source-card:`, so the schema deliberately omits `:source_entity` and `:referenced_entities`."[:map {:closed true}[:reasoning :string]- [:query :map]+ [:query construct/LLMExternalQuery][:title {:optional true} [:maybe :string]][:display {:optional true:description "Visualization type for displaying the query results in Slack. Required in practice whenever the user asks for a chart or graph, and it must match any requested chart type. Valid values: 'table', 'bar', 'line', 'pie', 'area', 'row', 'scatter', 'funnel'. Use requested chart types like 'line', 'bar', 'area', 'pie', 'scatter', 'funnel', 'row', or 'table' when they fit the query. Omitting this field falls back to Metabase's default table display, so do not omit it for chart or graph requests. Only omit it when you intentionally want a plain table and the user did not request a chart type."}diff --git a/src/metabase/metabot/tools/sql/create.clj b/src/metabase/metabot/tools/sql/create.cljindex 3f5ced0f87b..a9b147c5323 100644--- a/src/metabase/metabot/tools/sql/create.clj+++ b/src/metabase/metabot/tools/sql/create.clj@@ -58,7 +58,10 @@- :query-id - The ID of the created query- :query-content - The SQL content- :database - Database ID"- [{:keys [database-id sql]}]+ [{:keys [database-id sql]} :- [:map {:closed true}+ [:database-id :int]+ [:sql :string]+ [:name {:optional true} [:maybe :string]]]](log/info "Creating SQL query"{:database-id database-id:sql-length (count sql)})diff --git a/src/metabase/metabot/tools/sql/edit.clj b/src/metabase/metabot/tools/sql/edit.cljindex 1fdd3af0ae9..86ab44676a5 100644--- a/src/metabase/metabot/tools/sql/edit.clj+++ b/src/metabase/metabot/tools/sql/edit.clj@@ -2,6 +2,7 @@"Tool for editing existing SQL queries."(:require[clojure.string :as str]+ [metabase.metabot.schema :as metabot.schema][metabase.metabot.tools.sql.common :as metabot.tools.sql.common][metabase.metabot.tools.sql.validation :as metabot.tools.sql.validation][metabase.metabot.util :as metabot.u]@@ -47,7 +48,15 @@- description: New description for the query (optional)Returns an `operation-result` map. For details see its docstring."- [{:keys [query-id edits queries-state]}]+ [{:keys [query-id edits queries-state]}+ :- [:map {:closed true}+ [:query-id [:or :string :int]]+ [:edits [:sequential [:map {:closed true}+ [:old_string :string]+ [:new_string :string]+ [:replace_all {:optional true} [:maybe :boolean]]]]]+ [:checklist {:optional true} [:maybe :string]]+ [:queries-state [:map-of :string ::metabot.schema/query]]]](log/info "Editing SQL query" {:query-id query-id :edit-count (count edits)});; Look up query from in-memory statediff --git a/src/metabase/metabot/tools/sql/replace.clj b/src/metabase/metabot/tools/sql/replace.cljindex 39aab8ea0bb..d9742a31f88 100644--- a/src/metabase/metabot/tools/sql/replace.clj+++ b/src/metabase/metabot/tools/sql/replace.clj@@ -1,6 +1,7 @@(ns metabase.metabot.tools.sql.replace"Tool for replacing SQL query content entirely while preserving metadata."(:require+ [metabase.metabot.schema :as metabot.schema][metabase.metabot.tools.sql.common :as metabot.tools.sql.common][metabase.metabot.tools.sql.validation :as metabot.tools.sql.validation][metabase.util.i18n :refer [tru]]@@ -25,7 +26,14 @@- :query-id - The ID of the updated query- :query-content - The new SQL content- :database - Database ID"- [{:keys [query-id sql queries-state]}]+ [{:keys [query-id sql queries-state]}+ :- [:map {:closed true}+ [:query-id [:or :string :int]]+ [:sql :string]+ [:checklist {:optional true} [:maybe :string]]+ [:name {:optional true} [:maybe :string]]+ [:description {:optional true} [:maybe :string]]+ [:queries-state [:map-of :string ::metabot.schema/query]]]](log/info "Replacing SQL query" {:query-id query-id :sql-length (count sql)});; Look up query from in-memory statediff --git a/src/metabase/metabot/tools/subscriptions.clj b/src/metabase/metabot/tools/subscriptions.cljindex bf63ac92c92..16a9493f659 100644--- a/src/metabase/metabot/tools/subscriptions.clj+++ b/src/metabase/metabot/tools/subscriptions.clj@@ -33,7 +33,10 @@(-> cardapi/read-check(select-keys [:id :name :collection_id :description :display :parameter_mappings])- (assoc :dashboard_card_id id :dashboard_id dashboard-id)))+ (assoc :dashboard_card_id id+ :dashboard_id dashboard-id+ :include_csv false+ :include_xls false)))channel-name (some->> slack-channelchannel.settings/find-cached-slack-channel-or-username;; match existing code which stores display names like "#some-channel"@@ -82,7 +85,7 @@[:dashboard_id :int][:email {:optional true} [:maybe :string]][:slack_channel {:optional true} [:maybe :string]]- [:schedule [:map+ [:schedule [:map {:closed true}[:frequency [:enum "hourly" "daily" "weekly" "monthly"]][:hour {:optional true} [:maybe :int]][:day_of_week {:optional true} [:maybe :string]]diff --git a/src/metabase/metabot/tools/transforms.clj b/src/metabase/metabot/tools/transforms.cljindex 46e3cece9fe..bd27b119a76 100644--- a/src/metabase/metabot/tools/transforms.clj+++ b/src/metabase/metabot/tools/transforms.clj@@ -137,7 +137,7 @@[:maybe :int]][:edit_action{:description "You MUST set this param. Use new_content and edits according to mode you choose."}- [:map+ [:map {:closed true}[:mode{:description (str "Use 'edit' mode for targeted string replacements. ""Use 'replace' mode to replace entire content.")}@@ -145,7 +145,7 @@[:edits{:optional true:description "List of targeted string replacements to apply sequentially"}- [:maybe [:sequential [:map+ [:maybe [:sequential [:map {:closed true}[:old_string :string][:new_string :string][:replace_all {:optional true} [:maybe :boolean]]]]]]@@ -214,7 +214,7 @@[:maybe :int]][:edit_action{:description "You MUST set this param. Use new_content and edits according to mode you choose."}- [:map+ [:map {:closed true}[:mode{:description (str "Use 'edit' mode for targeted string replacements. ""Use 'replace' mode to replace entire content.")}@@ -222,7 +222,7 @@[:edits{:optional true:description "List of targeted string replacements to apply sequentially"}- [:maybe [:sequential [:map+ [:maybe [:sequential [:map {:closed true}[:old_string :string][:new_string :string][:replace_all {:optional true} [:maybe :boolean]]]]]]diff --git a/src/metabase/metabot/tools/util.clj b/src/metabase/metabot/tools/util.cljindex 1f4e3342f9f..409d00a1069 100644--- a/src/metabase/metabot/tools/util.clj+++ b/src/metabase/metabot/tools/util.clj@@ -93,7 +93,7 @@(-> {:field_id field-id:name (or (:lib/desired-column-alias column)(:lib/source-column-alias column))- :display_name (lib/display-name query column)+ :display_name (lib/display-name query (dissoc column :table-reference)):type (convert-field-type column)}(m/assoc-some :description (:description column):base_type base-typediff --git a/src/metabase/metrics/dimension.clj b/src/metabase/metrics/dimension.cljindex 0e33fe9242f..21d2dc07e8c 100644--- a/src/metabase/metrics/dimension.clj+++ b/src/metabase/metrics/dimension.clj@@ -55,7 +55,7 @@"Encode-time defaulting policy for a dimension: `:display-name` falls back to `:name`, the fivealways-on-the-wire keys are present even when nil, nil-valued optional keys are dropped."[dim]- (-> (merge {:id nil :name nil :effective-type nil :semantic-type nil} dim)+ (-> (merge {:id nil :name nil :effective-type nil :semantic-type nil} (dissoc dim :lib/source))(as-> dim (update dim :display-name #(or % (:name dim))))(drop-nil-keys [:has-field-values :status :status-message :dimension-interestingness :group:sources :default-temporal-unit])))@@ -210,7 +210,7 @@Returns the metrics seq with `:dimensions` updated in place. `metrics` is thelast arg so this composes cleanly with `->>` pipelines."[field-cols :- [:sequential :keyword]- metrics :- [:sequential :map]]+ metrics :- [:sequential :metabase.queries.schema/card]];; Resolve each (metric, dimension) pair exactly once and carry the field id alongside its;; dimension, so the merge pass below reads it instead of resolving all over again.(let [dims+fids (mapv (fn [metric]@@ -235,8 +235,8 @@(mu/defn dimension-values :- ms/FieldValuesResult"Fetch values for a dimension given its UUID.Uses the same logic as the field values API."- [dimensions :- [:maybe [:sequential :map]]- dimension-mappings :- [:maybe [:sequential :map]]+ [dimensions :- [:maybe [:sequential ::lm.schema/dimension]]+ dimension-mappings :- [:maybe [:sequential ::lm.schema/dimension-mapping]]dimension-id :- :string](let [field-id (lib-metric/resolve-dimension-to-field-id dimensions dimension-mappings dimension-id)field (metrics.db/field field-id)]@@ -244,8 +244,8 @@(mu/defn dimension-search-values :- [:sequential [:vector :string]]"Search for values of a dimension that contain the query string."- [dimensions :- [:maybe [:sequential :map]]- dimension-mappings :- [:maybe [:sequential :map]]+ [dimensions :- [:maybe [:sequential ::lm.schema/dimension]]+ dimension-mappings :- [:maybe [:sequential ::lm.schema/dimension-mapping]]dimension-id :- :stringquery-string :- ms/NonBlankString](let [field-id (lib-metric/resolve-dimension-to-field-id dimensions dimension-mappings dimension-id)]@@ -254,8 +254,8 @@(mu/defn dimension-remapped-value :- [:or [:tuple :any] [:tuple :any :string]]"Get the remapped value for a specific dimension value.Returns a pair like [value display-name] if remapping exists, or just [value] otherwise."- [dimensions :- [:maybe [:sequential :map]]- dimension-mappings :- [:maybe [:sequential :map]]+ [dimensions :- [:maybe [:sequential ::lm.schema/dimension]]+ dimension-mappings :- [:maybe [:sequential ::lm.schema/dimension-mapping]]dimension-id :- :stringvalue :- :string](let [field-id (lib-metric/resolve-dimension-to-field-id dimensions dimension-mappings dimension-id)diff --git a/src/metabase/metrics/transforms.clj b/src/metabase/metrics/transforms.cljindex eac7d0ac537..59da3b1880b 100644--- a/src/metabase/metrics/transforms.clj+++ b/src/metabase/metrics/transforms.clj@@ -14,7 +14,25 @@(mu/defn normalize-dimension :- ::lib-metric.schema/persisted-dimension"Normalize a dimension after JSON parsing. Keys are kebab-case and type values are keywords.See [[lib-metric.schema/persisted-dimension]]."- [dim :- :map]+ [dim :- [:map {:closed true}+ [:id [:maybe ::lib-metric.schema/dimension-id]]+ [:lib/source {:optional true} [:maybe [:or :keyword :string]]]+ [:name {:optional true} [:maybe :string]]+ [:display-name {:optional true} [:maybe :string]]+ [:description {:optional true} [:maybe :string]]+ [:effective-type {:optional true} [:maybe [:or :string :keyword]]]+ [:semantic-type {:optional true} [:maybe [:or :string :keyword]]]+ [:has-field-values {:optional true} [:maybe [:or :string :keyword]]]+ [:status {:optional true} [:maybe [:or :string :keyword]]]+ [:status-message {:optional true} [:maybe :string]]+ [:sources {:optional true} [:maybe [:sequential+ [:map {:closed true}+ [:type [:or :string :keyword]]+ [:field-id {:optional true} [:maybe :int]]+ [:binning {:optional true} [:maybe :boolean]]]]]]+ [:group {:optional true} [:maybe ::lib-metric.schema/dimension-group]]+ [:default-temporal-unit {:optional true} [:or :string :keyword]]+ [:default {:optional true} [:maybe :boolean]]]](cond-> dim(:status dim) (update :status keyword)(:effective-type dim) (update :effective-type keyword)@@ -42,6 +60,7 @@:out (fn [dims](some->> dimsmi/json-out-with-keywordization+ (remove nil?)(mapv normalize-dimension)))})(def transform-dimension-mappingsdiff --git a/src/metabase/model_persistence/models/persisted_info.clj b/src/metabase/model_persistence/models/persisted_info.cljindex 668f3f6faf1..5746d79161d 100644--- a/src/metabase/model_persistence/models/persisted_info.clj+++ b/src/metabase/model_persistence/models/persisted_info.clj@@ -56,24 +56,16 @@{:field-name field-name:base-type (or effective_type base_type)})-(def ^:private Metadata- "Spec for metadata. Just asserting we have base types and names, not the full metadata of the qp."- [:maybe- [:sequential- [:map- [:name :string]- [:base_type ::lib.schema.common/base-type]- [:effective_type {:optional true} ::lib.schema.common/base-type]]]])-(mu/defn metadata->definition :- ::lib.schema.metadata/persisted-info.definition"Returns a ddl definition datastructure. A :table-name and :field-deifinitions vector of field-name and base-type."- [metadata :- Metadata table-name]+ [metadata :- [:maybe ::lib.schema.metadata/card.result-metadata]+ table-name :- ::lib.schema.common/non-blank-string]{:table-name table-name:field-definitions (mapv field-metadata->field-defintion metadata)})(mu/defn query-hash"Base64 string of the hash of a query."- [query :- :map]+ [query :- :metabase.lib-be.schema/maybe-legacy-or-empty-query](String. ^bytes (codecs/bytes->b64 (qp.util/query-hash query))))(def ^:dynamic *allow-persisted-substitution*diff --git a/src/metabase/model_persistence/schema.clj b/src/metabase/model_persistence/schema.cljindex b0a0cd845ce..4691eaa21b9 100644--- a/src/metabase/model_persistence/schema.clj+++ b/src/metabase/model_persistence/schema.clj@@ -2,31 +2,20 @@"Malli schemas for the model-persistence module."(:require[metabase.lib.schema.id :as lib.schema.id]+ [metabase.lib.schema.metadata :as lib.schema.metadata][metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms]))(mr/def ::persisted-info.definition"The `:definition` column of a PersistedInfo, decoded."- :map)+ ::lib.schema.metadata/persisted-info.definition)(mr/def ::persisted-info"A PersistedInfo as selected from the app DB: every column of `:persisted_info`."- [:map {:closed true}- [:id ms/PositiveInt]- [:database_id ::lib.schema.id/database]- [:card_id [:maybe ::lib.schema.id/card]]- [:question_slug :string]- [:table_name :string]- [:definition [:maybe ::persisted-info.definition]]- [:query_hash [:maybe :string]]- [:active :boolean]- [:state :string]- [:refresh_begin ms/TemporalInstant]- [:refresh_end [:maybe ms/TemporalInstant]]- [:state_change_at [:maybe ms/TemporalInstant]]- [:error [:maybe :string]]- [:created_at ms/TemporalInstant]- [:creator_id [:maybe ::lib.schema.id/user]]])+ [:merge+ ::persisted-info.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::persisted-info.update"What an update (or insert) of a PersistedInfo accepts: every column of `:persisted_info` except `id`, all optional."diff --git a/src/metabase/model_persistence/task/persist_refresh.clj b/src/metabase/model_persistence/task/persist_refresh.cljindex 3cfd20ac66b..ba3245fe166 100644--- a/src/metabase/model_persistence/task/persist_refresh.clj+++ b/src/metabase/model_persistence/task/persist_refresh.clj@@ -9,6 +9,7 @@[metabase.driver :as driver][metabase.driver.ddl.interface :as ddl.i][metabase.events.core :as events]+ [metabase.lib.schema.id :as lib.schema.id][metabase.model-persistence.db :as model-persistence.db][metabase.model-persistence.models.persisted-info :as persisted-info][metabase.model-persistence.settings :as model-persistence.settings]@@ -20,6 +21,7 @@[metabase.util :as u][metabase.util.log :as log][metabase.util.malli :as mu]+ [metabase.util.malli.schema :as ms][potemkin.types :as p][toucan2.core :as t2])(:import@@ -85,10 +87,20 @@[results](some-> results :error-details seq))+(def ^:private TaskDetails+ [:map {:closed true}+ [:success :int]+ [:error :int]+ [:trigger [:maybe (ms/InstanceOfClass org.quartz.Trigger)]]+ [:error-details {:optional true} [:sequential [:map {:closed true}+ [:persisted-info-id ms/PositiveInt]+ [:error {:optional true} [:maybe :string]]]]]])+(mu/defn- publish-refresh-error-event!"Fire off an event that will eventually send an email to the admin if there are any errors in the persisted modelrefresh task."- [db-id task-details]+ [db-id :- [:maybe ::lib.schema.id/database]+ task-details :- TaskDetails](try(let [error-details (error-details task-details)error-details-by-id (m/index-by :persisted-info-id error-details)diff --git a/src/metabase/models/db.clj b/src/metabase/models/db.cljindex dd3f556a189..675c9058d28 100644--- a/src/metabase/models/db.clj+++ b/src/metabase/models/db.clj@@ -3,8 +3,10 @@additional logic, so no other namespace in the module runs a query itself (the model machinery still uses`toucan2.core`)."(:require+ [malli.core :as mc][metabase.lib.schema.id :as lib.schema.id][metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms][metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay][toucan2.core :as t2]@@ -44,14 +46,99 @@value :- [:maybe [:or :string :int :boolean :keyword]]](t2/select-one-pk model field value))-(mu/defn after-select-via-identity-query- "`row-map` run through the after-select machinery of `model`.+(def ^:private model-row-schema+ "The literal registry keyword of the row/update schema of each model the generic entity helpers below are called+ with (a literal keyword, not a `require`, to avoid a dependency cycle with every module that owns one of these+ models)."+ {:model/Action :metabase.actions.schema/action.for-update+ :model/Card :metabase.queries.schema/card.update+ :model/Channel :metabase.channel.schema/channel.update+ :model/ChannelTemplate :metabase.channel.schema/channel-template.update+ :model/Collection :metabase.collections.schema/collection.update+ :model/CustomVizPlugin :metabase-enterprise.custom-viz-plugin.schema/custom-viz-plugin.update+ :model/Dashboard :metabase.dashboards.schema/dashboard.update+ :model/DashboardCard :metabase.dashboards.schema/dashboard-card.update+ :model/DashboardCardSeries :metabase.dashboards.schema/dashboard-card-series.update+ :model/DashboardTab :metabase.dashboards.schema/dashboard-tab.update+ :model/Database :metabase.warehouses.schema/database.update+ :model/Dimension :metabase.warehouse-schema.schema/dimension.update+ :model/Document :metabase.documents.schema/document.update+ :model/EmbeddingTheme :metabase.embedding.schema/embedding-theme.update+ :model/Exploration :metabase.explorations.schema/exploration.update+ :model/Field :metabase.warehouse-schema.schema/field.update+ :model/FieldUserSettings :metabase.warehouse-schema.schema/field-user-settings.update+ :model/FieldValues :metabase.warehouse-schema.schema/field-values.update+ :model/Glossary :metabase.glossary.schema/glossary.update+ :model/HTTPAction :metabase.actions.schema/httpaction.update+ :model/ImplicitAction :metabase.actions.schema/implicit-action.update+ :model/Measure :metabase.measures.schema/measure.update+ :model/Metabot :metabase.metabot.schema/metabot.update+ :model/MetabotPrompt :metabase.metabot.schema/metabot-prompt.update+ :model/NativeQuerySnippet :metabase.native-query-snippets.schema/native-query-snippet.update+ :model/Notification :metabase.notification.schema/notification.update+ :model/NotificationCard :metabase.notification.schema/notification-card.update+ :model/NotificationHandler :metabase.notification.schema/notification-handler.update+ :model/NotificationRecipient :metabase.notification.schema/notification-recipient.update+ :model/NotificationSubscription :metabase.notification.schema/notification-subscription.update+ :model/OsiAiContext :metabase.osi.schema/osi-ai-context.update+ :model/PythonLibrary :metabase-enterprise.transforms-python.schema/python-library.update+ :model/QueryAction :metabase.actions.schema/query-action.update+ :model/Segment :metabase.segments.schema/segment.update+ :model/Table :metabase.warehouse-schema.schema/table.update+ :model/TableIndex :metabase.indexes.schema/table-index.update+ :model/Timeline :metabase.timeline.schema/timeline.update+ :model/TimelineEvent :metabase.timeline.schema/timeline-event.update+ :model/Transform :metabase.transforms.schema/transform.update+ :model/TransformJob :metabase.transforms.schema/transform-job.update+ :model/TransformJobTransformTag :metabase.transforms.schema/transform-job-transform-tag.update+ :model/TransformTag :metabase.transforms.schema/transform-tag.update+ :model/TransformTransformTag :metabase.transforms.schema/transform-transform-tag.update})++(defn- model-schema-key [prefix model]+ (keyword "metabase.models.db" (str prefix "." (name model))))++(doseq [[model schema] model-row-schema]+ (mr/register! (model-schema-key "model-row" model)+ [:map {:closed true} [:model [:= model]] [:row schema]])+ (mr/register! (model-schema-key "model-rows" model)+ [:map {:closed true} [:model [:= model]] [:rows [:sequential schema]]]))++(mr/def ::unregistered-model-row+ "A row of a model not listed in [[model-row-schema]], such as the test-double models `metabase.models.util.spec-update`+ specs are tested with; its keys are that model's."+ [:map {:closed false, ::mr/deliberately-open true, :description "row of an unregistered model"}])++(mr/def ::model-row+ "A `{:model ..., :row ...}` pair naming one of the models the generic entity helpers below are called with, the+ row typed by that model's own update schema."+ (conj (into [:multi {:dispatch :model, :lazy-refs true}]+ (for [model (keys model-row-schema)]+ [model (model-schema-key "model-row" model)]))+ [::mc/default [:map {:closed true} [:model :keyword] [:row ::unregistered-model-row]]]))++(mr/def ::model-rows+ "Like `::model-row`, but for a batch of rows of the same model."+ (conj (into [:multi {:dispatch :model, :lazy-refs true}]+ (for [model (keys model-row-schema)]+ [model (model-schema-key "model-rows" model)]))+ [::mc/default [:map {:closed true} [:model :keyword] [:rows [:sequential ::unregistered-model-row]]]]))++(mr/def ::after-select-row+ "A row not yet run through its model's after-select, e.g. a stored revision snapshot whose keys older Metabase versions own."+ [:map {:closed false, ::mr/deliberately-open true,+ :description "shape depends on the model, hydration state, and revision history"}])++(def ^:private AfterSelectEntity+ "A `{:model ..., :row ...}` pair naming any model (including test-only ones derived from `:metabase/model`, and+ ones outside [[model-row-schema]]) and a row of arbitrary shape to run through its after-select machinery."+ [:map {:closed true}+ [:model [:or :keyword symbol?]]+ [:row ::after-select-row]])- `row-map` is a model instance row and so is generic across models; typed as a plain keyword-keyed map- rather than a closed schema."- [model :- [:or :keyword symbol?]- row-map :- [:map-of :keyword [:maybe :some]]]- (t2/select-one model (t2.identity-query/identity-query [row-map])))+(mu/defn after-select-via-identity-query+ "`entity`'s `:row` run through the after-select machinery of `entity`'s `:model`."+ [entity :- AfterSelectEntity]+ (t2/select-one (:model entity) (t2.identity-query/identity-query [(:row entity)])))(mu/defn entities-reducible"A reducible of the `model` rows whose `filter-column` is one of `filter-ids` (every row when `filter-column` is@@ -84,35 +171,25 @@(seq order-columns) (assoc :order-by (mapv (fn [column] [column :asc]) order-columns)))))(mu/defn update-entity!- "Apply `changes` to the `model` row with `id`, returning the number updated.-- Generic across models, so `changes` is typed as a plain keyword-keyed map rather than a closed schema."- [model :- [:or :keyword symbol?]- id :- [:or :int :string]- changes :- [:map-of :keyword [:maybe :some]]]- (t2/update! model id changes))+ "Apply `entity`'s `:row` (a column diff) to `entity`'s `:model` row with `id`, returning the number updated."+ [id :- [:or :int :string]+ entity :- ::model-row]+ (t2/update! (:model entity) id (:row entity)))(mu/defn insert-entity!- "Insert the `model` `row` and return the inserted instance.-- Generic across models, so `row` is typed as a plain keyword-keyed map rather than a closed schema."- [model :- [:or :keyword symbol?]- row :- [:map-of :keyword [:maybe :some]]]- (t2/insert-returning-instance! model row))+ "Insert `entity`'s `:row` into `entity`'s `:model` and return the inserted instance."+ [entity :- ::model-row]+ (t2/insert-returning-instance! (:model entity) (:row entity)))(mu/defn insert-entity-returning-pk!- "Insert the `model` `row` and return its primary key.-- Generic across models, so `row` is typed as a plain keyword-keyed map rather than a closed schema."- [model :- [:or :keyword symbol?]- row :- [:map-of :keyword [:maybe :some]]]- (t2/insert-returning-pk! model row))+ "Insert `entity`'s `:row` into `entity`'s `:model` and return its primary key."+ [entity :- ::model-row]+ (t2/insert-returning-pk! (:model entity) (:row entity)))(mu/defn insert-entities!- "Insert the `model` `rows`, returning the number inserted."- [model :- [:or :keyword symbol?]- rows :- [:sequential :map]]- (t2/insert! model rows))+ "Insert `entities`'s `:rows` into `entities`'s `:model`, returning the number inserted."+ [entities :- ::model-rows]+ (t2/insert! (:model entities) (:rows entities)))(mu/defn delete-entity!"Delete the `model` row with `id`, returning the number deleted."diff --git a/src/metabase/models/interface.clj b/src/metabase/models/interface.cljindex 29f7f062e63..98d03f24a69 100644--- a/src/metabase/models/interface.clj+++ b/src/metabase/models/interface.clj@@ -256,23 +256,23 @@(mu/defn assert-enum"Assert that a value is one of the values in `enum`."- [enum :- [:set :any]- value]+ [enum :- [:set [:maybe [:or :keyword :string]]]+ value :- [:maybe [:or :keyword :string]]](when-not (contains? enum value)(throw (ex-info (format "Invalid value %s. Must be one of %s" value (str/join ", " enum)) {:status-code 400:value value}))))(mu/defn assert-optional-enum"Assert that a value is one of the values in `enum` or `nil`."- [enum :- [:set :any]- value :- :any]+ [enum :- [:set [:or :keyword :string]]+ value :- [:maybe [:or :keyword :string]]](when (some? value)(assert-enum enum value)))(mu/defn assert-namespaced"Assert that a value is a namespaced keyword under `qualified-ns`."[qualified-ns :- string?- value]+ value :- [:or :keyword :string]](when-not (= qualified-ns (-> value keyword namespace))(throw (ex-info (format "Must be a namespaced keyword under :%s, got: %s" qualified-ns value) {:status-code 400:value value}))))@@ -622,7 +622,7 @@{:pre [(map? row-map)]}(let [model (t2/resolve-model modelable)](try- (models.db/after-select-via-identity-query model row-map)+ (models.db/after-select-via-identity-query {:model model :row row-map})(catch Throwable e(throw (ex-info (format "Error doing after-select for model %s: %s" model (ex-message e)){:model model}@@ -778,7 +778,8 @@([fn-symb :- qualified-symbol?read-or-write :- [:enum :read :write]- object :- :map]+ object :- [:maybe [:fn {:error/message "a Toucan instance or model-like record"}+ #(some? (t2.protocols/model %))]]](and object(check-perms-with-fn fn-symb (perms-objects-set object read-or-write))))@@ -869,6 +870,11 @@[_original-model dest-key _hydrated-key][(u/->snake_case_en (keyword (str (name dest-key) "_id")))])+(mr/def ::hydratable-item+ "An item to hydrate: usually a Toucan instance, but callers may pass a plain map (or nil, for a sparse position)+ since this helper only assocs a hydration key onto it."+ [:maybe [:map {:closed false, ::mr/deliberately-open true, :description "a Toucan instance or a plain map"}]])+(mu/defn instances-with-hydrated-data;; TODO: this example is wrong, we don't get a vector of tables"Helper function to write batched hydrations.@@ -883,11 +889,13 @@{:id 2 :tables [...tables-from-db-2]}]- key->hydrated-items-fn: is a function that returns a map with key is `instance-key` and value is the hydrated data of that instance."- [instances :- [:sequential :any]+ [instances :- [:sequential ::hydratable-item]hydration-key :- :keywordinstance-key->hydrated-data-fn :- fn?instance-key :- :keyword- & [{:keys [default] :as _options}]]+ & [{:keys [default] :as _options}] :- [:* [:map {:closed true}+ [:default {:optional true}+ [:maybe [:or :boolean number? [:map {:closed true}] [:= []]]]]]]](when (seq instances)(let [key->hydrated-items (instance-key->hydrated-data-fn)](for [item instances]diff --git a/src/metabase/models/serialization.clj b/src/metabase/models/serialization.cljindex 7dae9ebdd32..860cb734479 100644--- a/src/metabase/models/serialization.clj+++ b/src/metabase/models/serialization.clj@@ -74,6 +74,7 @@[metabase.models.serialization.resolve :as resolve][metabase.models.serialization.resolve.default :as resolve.default][metabase.models.visualization-settings :as mb.viz]+ [metabase.parameters.schema][metabase.util :as u][metabase.util.date-2 :as u.date][metabase.util.json :as json]@@ -81,6 +82,7 @@[metabase.util.malli :as mu][metabase.util.malli.humanize :as mu.humanize][metabase.util.malli.registry :as mr]+ [metabase.util.malli.schema :as ms][metabase.util.match :as match][potemkin :as p][toucan2.core :as t2]@@ -648,7 +650,7 @@pk (first (t2/primary-keys model))id (get local pk)](log/tracef "Upserting %s %d" model-name id)- (models.db/update-entity! model id ingested)+ (models.db/update-entity! id (lib/normalize :metabase.models.db/model-row {:model model :row ingested}))(models.db/entity-by-pk model pk id)))(defmulti load-insert!@@ -670,7 +672,7 @@(defmethod load-insert! :default [model-name ingested](log/tracef "Inserting %s" model-name)- (models.db/insert-entity! (t2.model/resolve-model (symbol model-name)) ingested))+ (models.db/insert-entity! (lib/normalize :metabase.models.db/model-row {:model (t2.model/resolve-model (symbol model-name)) :row ingested})))(defmulti load-one!"Black box for integrating a deserialized entity into this appdb.@@ -849,7 +851,7 @@Throws if the corresponding entity cannot be found.Unusual parameter order means this can be used as `(update x :some_id import-fk 'SomeModel)`."- [eid+ [eid :- [:maybe [:or :string [:sequential :string]]]model :- :metabase.models.serialization.path/model-keyword-or-symbol](resolve/import-fk (import-resolver) eid model))@@ -860,9 +862,9 @@Unusual parameter order lets this be called as, for example, `(update x :db_id *export-fk-keyed* :model/Database :name)`.Note: This assumes the primary key is called `:id`."- [id+ [id :- [:maybe ms/PositiveInt]model :- :metabase.models.serialization.path/model-keyword-or-symbol- field]+ field :- :keyword](resolve/export-fk-keyed (export-resolver) id model field))(defn ^:dynamic *import-fk-keyed*@@ -993,9 +995,16 @@;;; ## MBQL Fields+(mr/def ::mbql-node+ "Any node reached while walking an MBQL form being exported or imported, which may or may not be an MBQL clause."+ [:schema {::mr/deliberately-open true, :description "an MBQL form node"} :any])++(def ^:private MBQLNode+ [:ref ::mbql-node])+(mu/defn- mbql-ref? :- [:maybe [:enum :field :field-id :dimension :metric :segment :measure]]"Is given form an MBQL entity reference?"- [form]+ [form :- MBQLNode](when (and (vector? form)(#{:field :field-id :dimension :metric :segment :measure} (keyword (first form))))(keyword (first form))))@@ -1027,7 +1036,7 @@(def ^:private ^:dynamic *required-lib-uuids-for-export* nil)(mu/defn- collect-required-lib-uuids :- [:set ::lib.schema.common/uuid]- [x]+ [x :- MBQLNode](set(match/match-many x[:aggregation (_opts :guard map?) (uuid :guard string?)]@@ -1429,9 +1438,11 @@(mu/defn export-parameters"Given the :parameter field of a `Card` or `Dashboard`, as a vector of maps, convertsit to a portable form with the CardIds/FieldIds replaced with `[db schema table field]` references.- Parameters are sorted by `:id` for stable serialization output. A `:position` field is added- to preserve display order through the sort."- [parameters :- [:maybe [:sequential :map]]]+ Parameters are sorted by `:id` for stable serialization output (a nil `:id` sorts first). A `:position` field+ is added to preserve display order through the sort."+ [parameters :- [:maybe [:sequential+ [:merge :metabase.parameters.schema/parameter-with-optional-type+ [:map [:id {:optional true} [:maybe :metabase.lib.schema.parameter/id]]]]]]](->> parameters(map-indexed (fn [i p] (assoc p :position i)))(sort-by :id)diff --git a/src/metabase/models/serialization/path.clj b/src/metabase/models/serialization/path.cljindex 785df762d75..87dd886b7c6 100644--- a/src/metabase/models/serialization/path.clj+++ b/src/metabase/models/serialization/path.clj@@ -76,7 +76,7 @@(mu/defn lookup-by-id"Given an entity ID string, finds the matching entity. This is useful when writing [[metabase.models.serialization/xform-one]] toturn a foreign key from a portable form to an appdb ID. Returns a Toucan entity or nil."- [model :- ::model-keyword-or-symbol id-str]+ [model :- ::model-keyword-or-symbol id-str :- :string](models.db/entity-by-entity-id model id-str))(defn field-hierarchydiff --git a/src/metabase/models/serialization/resolve.clj b/src/metabase/models/serialization/resolve.cljindex ba4ef1baec6..77d592cee92 100644--- a/src/metabase/models/serialization/resolve.clj+++ b/src/metabase/models/serialization/resolve.clj@@ -10,6 +10,7 @@[metabase.lib.schema.parameter :as lib.schema.parameter][metabase.models.visualization-settings :as mb.viz][metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr][metabase.util.match :as match]))(set! *warn-on-reflection* true)@@ -210,9 +211,13 @@([resolver exported](mbql-fully-qualified-names->ids* resolver exported)))+(mr/def ::mbql-node+ "Any node reached while walking an MBQL form being imported, which may or may not be an MBQL clause."+ [:schema {::mr/deliberately-open true, :description "an MBQL form node"} :any])+(mu/defn- mbql-clause-tag :- [:maybe [:enum :field :dimension :metric :segment :measure]]"Is given form an MBQL entity reference?"- [form]+ [form :- [:ref ::mbql-node]](when (and (vector? form)(#{:field :dimension :metric :segment :measure} (keyword (first form))))(keyword (first form))))diff --git a/src/metabase/models/util/spec_update.clj b/src/metabase/models/util/spec_update.cljindex db696025683..95312b120a3 100644--- a/src/metabase/models/util/spec_update.clj+++ b/src/metabase/models/util/spec_update.clj@@ -33,6 +33,10 @@(contains? row k))][k spec]))))+(mr/def ::generic-row+ "A row of arbitrary shape, whatever the caller-supplied `spec` describes; not fixed by this namespace."+ [:maybe [:map {:closed false, ::mr/deliberately-open true, :description "shape depends on the caller-supplied spec"}]])+(mr/def ::Spec[:schema {:registry {::spec [:map {:closed true}[:model :keyword]@@ -50,7 +54,7 @@;; other columns that are not part of compare-cols but are part of the table;; these columns will be added when create / update[:extra-cols {:optional true} [:sequential :keyword]]- [:nested-specs {:optional true} [:map-of :keyword [:ref ::spec]]]]}}+ [:nested-specs {:optional true} [:sequential [:tuple :keyword [:ref ::spec]]]]]}}[:ref ::spec]])(defn decode-spec@@ -139,7 +143,7 @@(sanitize-row row)))];; Update the parent row if needed(when (or force-update? updates-needed?)- (models.db/update-entity! model (id-col row) (sanitize-row row-with-refs)))+ (models.db/update-entity! (id-col row) {:model model :row (sanitize-row row-with-refs)}));; Handle non-ref nested models(when nested-specs(let [nested-without-refs (non-ref-nested-models nested-specs row)]@@ -167,7 +171,7 @@(let [;; Process any ref-in-parent models firstparent-updates (handle-refs-in-parent! nil row nested-specs path)row-with-refs (merge row parent-updates)- parent-id (models.db/insert-entity-returning-pk! model (sanitize-row row-with-refs))]+ parent-id (models.db/insert-entity-returning-pk! {:model model :row (sanitize-row row-with-refs)})](log/debugf "%s created a new entity %s %s" (format-path path) model parent-id);; Only process non-ref nested models(let [nested-without-refs (non-ref-nested-models nested-specs row)]@@ -180,7 +184,7 @@(do(log/tracef "%s no nested spec found, batch creating %d new rows of %s" (format-path path) (count to-create) model)(let [rows (map sanitize-row to-create)]- (models.db/insert-entities! model rows)))))+ (models.db/insert-entities! {:model model :rows rows})))))(when (seq to-delete);; TODO: cascade deletes?(log/debugf "%s deleting %d rows with ids %s" (format-path path) (count to-delete) (str/join ", " (map id-col to-delete)))@@ -229,7 +233,7 @@;; create(nil? existing-data)- (let [parent-id (models.db/insert-entity-returning-pk! model new-data-sanitized)+ (let [parent-id (models.db/insert-entity-returning-pk! {:model model :row new-data-sanitized})path (conj path parent-id)](log/debugf "%s Created a new entity %s %s" (format-path path) model parent-id)(handle-nested! nil new-data parent-id)@@ -241,7 +245,7 @@(log/debugf "%s ID changed from %s to %s - deleting and recreating"(format-path path) existing-id (id-col new-data))(models.db/delete-entity! model existing-id)- (let [parent-id (models.db/insert-entity-returning-pk! model new-data-sanitized)+ (let [parent-id (models.db/insert-entity-returning-pk! {:model model :row new-data-sanitized})path (conj path parent-id)](log/debugf "%s Created a new entity %s %s" (format-path path) model parent-id)(handle-nested! nil new-data parent-id)@@ -251,7 +255,7 @@(not= (compare-row new-data-sanitized) (compare-row existing-data-sanitized))(do(log/debugf "%s Updating" (format-path (conj path existing-id)))- (models.db/update-entity! model existing-id new-data-sanitized)+ (models.db/update-entity! existing-id {:model model :row new-data-sanitized})(handle-nested! existing-data new-data existing-id)existing-id)@@ -274,6 +278,8 @@(mu/defn do-update!"Update data in the database based on the diff between existing and new data.`spec` defines the structure of the data and how to compare it."- [existing-data new-data spec :- ::Spec]+ [existing-data :- ::generic-row+ new-data :- ::generic-row+ spec :- ::Spec](t2/with-transaction [](do-update!* existing-data new-data spec ["root"])))diff --git a/src/metabase/models/visualization_settings.cljc b/src/metabase/models/visualization_settings.cljcindex 87d1603de76..a3725832cca 100644--- a/src/metabase/models/visualization_settings.cljc+++ b/src/metabase/models/visualization_settings.cljc@@ -28,6 +28,7 @@[clojure.walk :as walk][malli.core :as mc][medley.core :as m]+ [metabase.lib.schema.common :as lib.schema.common][metabase.lib.schema.id :as lib.schema.id][metabase.util.log :as log][metabase.util.malli :as mu]@@ -37,12 +38,12 @@;;; -------------------------------------------------- Specs ---------------------------------------------------(def ^:private field-metadata-schema [:maybe map?])+(def ^:private field-metadata-schema [:maybe ::lib.schema.common/visualization-settings]);; field-str - a field reference that is a string, which could be a reference to some named field (ex: output of an;; aggregation) or to a fully qualified field name (in the context of serialization); we won't attempt to interpret it;; here, only report that it's a string and set it in the ref map appropriately(def ^:private column-ref-schema- [:map+ [:map {:closed true}[::field-id {:optional true} ::lib.schema.id/field][::column-name {:optional true} string?][::field-str {:optional true} string?]@@ -55,17 +56,17 @@[:tuple[:= "field"][:orn [:field-id ::lib.schema.id/field] [:field-str string?]]- [:orn [:field-metadata map?] [:nil nil?]]]]]+ [:orn [:field-metadata ::lib.schema.common/visualization-settings] [:nil nil?]]]]][:expression [:tuple [:= "ref"] [:tuple [:= "expression"] string?]]][:column-name [:tuple [:= "name"] string?]]]);; TODO: add more specific shape for this one-(def ^:private parameter-mapping-schema [:maybe map?])+(def ^:private parameter-mapping-schema [:maybe ::lib.schema.common/visualization-settings])(def ^:private click-behavior-schema- [:map+ [:map {:closed true}[::click-behavior-type {:optional true} keyword?]- [::link-type {:optional true} :any]+ [::link-type {:optional true} [:or [:= ::card] [:= ::dashboard] [:= ::url]]][::parameter-mapping {:optional true} parameter-mapping-schema][::link-template {:optional true} string?][::link-text {:optional true} string?]@@ -236,7 +237,7 @@passed the output of another fn (including, currently, `visualization-settings`). If the given `from-field-id`already has a click action, it will be replaced."{:added "0.40.0"}- [settings :- map?, col-key :- column-ref-schema, action :- click-behavior-schema]+ [settings :- ::lib.schema.common/visualization-settings, col-key :- column-ref-schema, action :- click-behavior-schema](-> settingswith-col-settings(update ::column-settings assoc col-key {::click-behavior action})))@@ -247,7 +248,7 @@(including, currently, `visualization-settings`). If the given `from-field-id` already has a click action, it willbe replaced."{:added "0.40.0"}- [settings :- map?+ [settings :- ::lib.schema.common/visualization-settingsfrom-field-id :- ::lib.schema.id/fieldto-entity-type :- entity-type-schemato-entity-id :- pos-int?diff --git a/src/metabase/native_query_snippets/models/native_query_snippet.clj b/src/metabase/native_query_snippets/models/native_query_snippet.cljindex 82408c7281a..b912d166dd6 100644--- a/src/metabase/native_query_snippets/models/native_query_snippet.clj+++ b/src/metabase/native_query_snippets/models/native_query_snippet.clj@@ -8,6 +8,7 @@[metabase.models.serialization :as serdes][metabase.native-query-snippets.db :as native-query-snippets.db][metabase.native-query-snippets.models.native-query-snippet.permissions :as snippet.perms]+ [metabase.native-query-snippets.schema][metabase.remote-sync.core :as remote-sync][metabase.util :as u][metabase.util.i18n :refer [deferred-tru tru]]diff --git a/src/metabase/native_query_snippets/schema.clj b/src/metabase/native_query_snippets/schema.cljindex b95d68d8627..7b69504e284 100644--- a/src/metabase/native_query_snippets/schema.clj+++ b/src/metabase/native_query_snippets/schema.clj@@ -1,24 +1,21 @@(ns metabase.native-query-snippets.schema(:require+ [metabase.collections.schema][metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.template-tag :as lib.schema.template-tag]+ [metabase.users.schema][metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms]))(mr/def ::native-query-snippet- "A NativeQuerySnippet as selected from the app DB: every column of `:native_query_snippet`."- [:map {:closed true}- [:id ::lib.schema.id/native-query-snippet]- [:name :string]- [:description [:maybe :string]]- [:content :string]- [:creator_id ::lib.schema.id/user]- [:archived :boolean]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]- [:collection_id [:maybe ::lib.schema.id/collection]]- [:entity_id :string]- [:template_tags [:maybe ::lib.schema.template-tag/template-tag-map]]])+ "A NativeQuerySnippet as selected from the app DB: every column of `:native_query_snippet`, plus `:creator` and+ `:collection` some callers hydrate onto it."+ [:merge+ ::native-query-snippet.update+ [:map {:closed true}+ [:id ::lib.schema.id/native-query-snippet]+ [:creator {:optional true} [:maybe :metabase.users.schema/user]]+ [:collection {:optional true} [:maybe :metabase.collections.schema/collection]]]])(mr/def ::native-query-snippet.update"What an update (or insert) of a NativeQuerySnippet accepts: every column of `:native_query_snippet` except `id`, all optional."diff --git a/src/metabase/notification/api/notification.clj b/src/metabase/notification/api/notification.cljindex 16e6277f926..d91d39f7952 100644--- a/src/metabase/notification/api/notification.clj+++ b/src/metabase/notification/api/notification.clj@@ -175,7 +175,7 @@(mu/defn create-notification! :- ::models.notification/FullyHydratedNotification"Create a notification with permission checks, hydration, email notifications, and event publishing."- [notification-info :- ::models.notification/FullyHydratedNotification]+ [notification-info :- ::CreateNotificationParams](api/create-check :model/Notification notification-info)(let [notification (models.notification/hydrate-notification(models.notification/create-notification!diff --git a/src/metabase/notification/db.clj b/src/metabase/notification/db.cljindex da99f648dd6..21b20421063 100644--- a/src/metabase/notification/db.clj+++ b/src/metabase/notification/db.clj@@ -7,6 +7,7 @@[honey.sql.helpers :as sql.helpers][malli.util :as mut][metabase.app-db.core :as mdb]+ [metabase.channel.schema :as channel.schema][metabase.lib.schema.id :as lib.schema.id][metabase.notification.schema :as notification.schema][metabase.util :as u]@@ -48,7 +49,7 @@[:type {:optional true} [:maybe [:or :keyword :string]]][:user_id {:optional true} [:maybe ::lib.schema.id/user]][:permissions_group_id {:optional true} [:maybe ms/PositiveInt]]- [:details {:optional true} [:maybe :map]]])+ [:details {:optional true} [:maybe ::notification.schema/notification-recipient.details]]])(def ^:private NotificationSubscriptionRow"A whole NotificationSubscription row for insert."@@ -61,10 +62,7 @@(def ^:private ChannelTemplateRow"A whole ChannelTemplate row for insert."- [:map {:closed true}- [:name {:optional true} [:maybe :string]]- [:channel_type {:optional true} [:maybe [:or :keyword :string]]]- [:details {:optional true} [:maybe :map]]])+ ::channel.schema/channel-template.update)(def ^:private AdminFilters"The filters accepted by [[admin-notifications-page]] and [[admin-notifications-count]]."@@ -678,7 +676,7 @@(mu/defn card"The Card with `card-id`, or nil."- [card-id :- ::lib.schema.id/card]+ [card-id :- [:maybe ::lib.schema.id/card]](t2/select-one :model/Card card-id))(mu/defn unarchived-carddiff --git a/src/metabase/notification/models.clj b/src/metabase/notification/models.cljindex 3c0c7547dcf..8f478c6637d 100644--- a/src/metabase/notification/models.clj+++ b/src/metabase/notification/models.clj@@ -5,12 +5,14 @@- more than one handlers where each handler has a channel, optionally a template, and more than one recpients."(:require[malli.core :as mc]+ [malli.util :as mut][medley.core :as m][metabase.channel.models.channel :as models.channel][metabase.lib.schema.common :as lib.schema.common][metabase.models.interface :as mi][metabase.models.util.spec-update :as models.u.spec-update][metabase.notification.db :as notification.db]+ [metabase.notification.schema :as notification.schema][metabase.notification.task.send-trigger :as notification.task.send-trigger][metabase.permissions.core :as perms][metabase.permissions.schema :as permissions.schema]@@ -58,8 +60,8 @@[:merge::users.schema/user[:map {:closed true}- [:date_joined RowTimestamp]- [:last_login RowTimestamp]]])+ [:date_joined {:optional true} RowTimestamp]+ [:last_login {:optional true} RowTimestamp]]])(def notification-types"Set of valid notification types."@@ -516,29 +518,29 @@;; the rest of the row. Authorization lives in the model's `before-update` hook (superuser-only).:compare-cols [:active :creator_id]:extra-cols [:payload_type :internal_id :payload_id]- :nested-specs {:payload {:model :model/NotificationCard- :compare-cols [:send_condition :send_once]- :extra-cols [:card_id]}- :subscriptions {:model :model/NotificationSubscription- :fk-column :notification_id- :compare-cols [:notification_id :type :event_name :cron_schedule :ui_display_type]- :multi-row? true}- :handlers {:model :model/NotificationHandler- :fk-column :notification_id- :compare-cols [:notification_id :channel_type :channel_id :template_id :active]- :multi-row? true- :nested-specs {:recipients {:model :model/NotificationRecipient- :fk-column :notification_handler_id- :compare-cols [:notification_handler_id :type :user_id :permissions_group_id :details]- :multi-row? true}- :template {:model :model/ChannelTemplate- :ref-in-parent :template_id- :compare-cols [:channel_type :name :details]}}}}})+ :nested-specs [[:payload {:model :model/NotificationCard+ :compare-cols [:send_condition :send_once]+ :extra-cols [:card_id]}]+ [:subscriptions {:model :model/NotificationSubscription+ :fk-column :notification_id+ :compare-cols [:notification_id :type :event_name :cron_schedule :ui_display_type]+ :multi-row? true}]+ [:handlers {:model :model/NotificationHandler+ :fk-column :notification_id+ :compare-cols [:notification_id :channel_type :channel_id :template_id :active]+ :multi-row? true+ :nested-specs [[:recipients {:model :model/NotificationRecipient+ :fk-column :notification_handler_id+ :compare-cols [:notification_handler_id :type :user_id :permissions_group_id :details]+ :multi-row? true}]+ [:template {:model :model/ChannelTemplate+ :ref-in-parent :template_id+ :compare-cols [:channel_type :name :details]}]]}]]})(defn- update-input-entries"Entries from `entries` whose key `spec` uses on update."[entries {:keys [compare-cols extra-cols nested-specs multi-row? id-col]}]- (let [allowed (into (set (concat compare-cols extra-cols (keys nested-specs)))+ (let [allowed (into (set (concat compare-cols extra-cols (map first nested-specs)));; multi-row rows are matched by their body-supplied id, so keep the id entry(when multi-row? [id-col]))](filterv (comp allowed first) entries)))@@ -547,7 +549,8 @@"::NotificationCard restricted to what the update spec writes - `:id` comes from the URL's notification."(into [:map {:closed true}](update-input-entries notification-card-entries- (get-in notification-update-spec [:nested-specs :payload]))))+ (second (m/find-first (comp #{:payload} first)+ (:nested-specs notification-update-spec))))));; ------------------------------------------------------------------------------------------------;;;; Helpers ;;@@ -649,21 +652,147 @@;; Public APIs ;;;; ------------------------------------------------------------------------------------------------;;+(mr/def ::event-topic+ "One of the event topics a `:notification/system-event` notification can be triggered for."+ [:enum+ :event/user-invited+ :event/notification-create+ :event/slack-token-invalid+ :event/comment-created+ :event/support-access-grant-created+ :event/transform-failed+ :event/transform-failure-digest+ :event/security-advisory-match])++(mr/def ::event-info.user-invited+ "The `:event_info` of an `:event/user-invited` system event: the invited User instance, plus the extra keys+ `create-and-invite-user!` stamps on it. `:object` is typed by instance, not by shape, since the extra keys are+ `assoc`ed onto the real Toucan row rather than replacing it."+ [:map {:closed true}+ [:object [:merge+ ::users.schema/user+ [:map {:closed true}+ [:is_from_setup {:optional true} [:maybe :boolean]]+ [:invite_method {:optional true} [:maybe :string]]+ [:invite_target {:optional true} [:maybe users.schema/InviteTarget]]+ [:sso_source {:optional true} [:maybe [:or :keyword :string]]]]]]+ [:details {:optional true}+ [:map {:closed true}+ [:invitor [:map {:closed true}+ [:email ms/Email]+ [:first_name {:optional true} [:maybe :string]]]]]]])++(mr/def ::event-info.security-advisory-match+ "The `:event_info` of an `:event/security-advisory-match` system event."+ [:map {:closed true}+ [:object [:map {:closed true}+ [:advisory_id [:string {:min 1}]]+ [:severity [:or :keyword :string]]+ [:title :string]+ [:description :string]+ [:match_status [:or :keyword :string]]+ [:advisory_url [:maybe :string]]+ [:remediation :string]+ [:affected_versions [:sequential [:map {:closed true}+ [:min [:re #"^\d+(?:\.\d+)*$"]]+ [:fixed [:re #"^\d+(?:\.\d+)*$"]]]]]]]])++(mr/def ::event-info.notification-create+ "The `:event_info` of an `:event/notification-create` system event."+ [:map {:closed true}+ [:object [:ref ::FullyHydratedNotification]]+ [:user-id [:maybe ms/PositiveInt]]])++(mr/def ::event-info.comment-created+ "The `:event_info` of an `:event/comment-created` system event."+ [:map {:closed true}+ [:entity_type :string]+ [:entity_title [:maybe :string]]+ [:comment_href :string]+ [:entity_href :string]+ [:created_at [:maybe ms/TemporalInstant]]+ [:author [:maybe :string]]+ [:comment [:maybe :string]]+ [:parent_author [:maybe :string]]+ [:parent_comment [:maybe :string]]+ [:style [:map {:closed true}+ [:color_text_dark :string]+ [:color_text_light :string]+ [:color_text_medium :string]]]+ [:email ms/Email]])++(mr/def ::event-info.support-access-grant-created+ "The `:event_info` of an `:event/support-access-grant-created` system event."+ [:map {:closed true}+ [:support_email [:maybe ms/Email]]+ [:ticket_number [:maybe :string]]+ [:duration_minutes :int]+ [:grant_end_time ms/TemporalInstant]+ [:password_reset_url :string]+ [:notes [:maybe :string]]])++(mr/def ::event-info.transform-failed+ "The `:event_info` of an `:event/transform-failed` system event."+ [:map {:closed true}+ [:email ms/Email]+ [:job_name [:maybe :string]]+ [:job_href :string]+ [:failure_count :int]+ [:skipped_count :int]+ [:failures [:sequential [:map {:closed true}+ [:transform_name [:maybe :string]]+ [:transform_href :string]+ [:message [:map {:closed true}+ [:first_line :string]+ [:details [:sequential :string]]]]]]]])++(mr/def ::event-info.transform-failure-digest+ "The `:event_info` of an `:event/transform-failure-digest` system event."+ [:map {:closed true}+ [:job_count :int]+ [:failure_count :int]+ [:jobs [:sequential [:map {:closed true}+ [:job_name [:maybe :string]]+ [:job_href :string]+ [:failure_count :int]+ [:first_failed [:maybe :string]]+ [:latest_error [:maybe :string]]]]]])++(def event-topic->event-info-schema+ "The `:event_info` schema published for each supported system event topic (a literal keyword, not a `require`,+ to avoid a dependency cycle with the module that publishes each event)."+ {:event/user-invited ::event-info.user-invited+ :event/notification-create ::event-info.notification-create+ :event/slack-token-invalid [:map {:closed true}]+ :event/comment-created ::event-info.comment-created+ :event/support-access-grant-created ::event-info.support-access-grant-created+ :event/transform-failed ::event-info.transform-failed+ :event/transform-failure-digest ::event-info.transform-failure-digest+ :event/security-advisory-match ::event-info.security-advisory-match})+(mr/def ::SystemEventPayload.request"The `:payload` of a system event notification as a client may send it: the event topic, plus the `:disable_links``POST /api/notification` stamps on. The event map itself is the server's to supply, so a request carries none."[:map {:closed true}- [:event_topic {:optional true} [:fn #(= "event" (-> % keyword namespace))]]+ [:event_topic {:optional true} ::event-topic][:disable_links {:optional true} [:maybe :boolean]]])(mr/def ::SystemEventPayload- "The `:payload` of a system event notification on its way to being sent: a [[::SystemEventPayload.request]] plus- the event map published under the topic, keyed the way [[metabase.events.core/publish-event!]] was handed it. A- `:notification/testing` notification's payload is empty."- [:merge- ::SystemEventPayload.request- [:map {:closed true}- [:event_info {:optional true} [:maybe [:map-of :keyword :any]]]]])+ "The `:payload` of a system event notification on its way to being sent: the event topic, plus the `:disable_links`+ `POST /api/notification` stamps on, plus the event map published under the topic, keyed the way+ [[metabase.events.core/publish-event!]] was handed it. A `:notification/testing` notification's payload is empty."+ (into [:multi {:dispatch :event_topic}]+ (concat+ (for [[topic info-schema] event-topic->event-info-schema]+ [topic [:map {:closed true}+ [:event_topic [:= topic]]+ [:disable_links {:optional true} [:maybe :boolean]]+ [:event_info {:optional true} [:maybe info-schema]]]])+ [[::mc/default+ [:map {:closed true}+ [:event_topic [:fn #(= "event" (-> % keyword namespace))]]+ [:disable_links {:optional true} [:maybe :boolean]]+ [:event_info {:optional true} [:maybe [:map {:closed true}]]]]]])))(defn hydrated-notification-schema"Schema for a notification hydrated with its creator, subscriptions and handlers, where each handler matches@@ -703,13 +832,39 @@[:merge::NotificationHandler[:map- [:template {:optional true} [:maybe ::models.channel/ChannelTemplate]]+ [:template {:optional true} [:maybe ::models.channel/ChannelTemplate]]+ [:channel {:optional true} [:maybe ::models.channel/Channel]]+ [:recipients {:optional true} [:sequential ::NotificationRecipient]]+ [:attachment_only {:optional true} [:maybe :boolean]]+ [:include_pdf {:optional true} [:maybe :boolean]]]]))++(def ^:private NotificationWithInlinePayload+ "A Notification row as `send-notification!` is handed it before hydration: with the system event `:payload`+ `metabase.notification.events.notification` assocs on, or the `:triggering_subscription`+ `metabase.notification.task.send` assocs on."+ (mut/merge ::notification.schema/notification+ [:map {:closed true}+ [:payload {:optional true} [:maybe ::SystemEventPayload]]+ [:triggering_subscription {:optional true} [:maybe ::notification.schema/notification-subscription]]]))++(def ^:private NotificationWithRawHandlers+ "An unsaved Notification as `POST /api/notification/send` hands it to `send-notification!`: no id or timestamps of+ its own, but the raw (pre-hydration) `:handlers` and `:subscriptions` straight off the request body."+ (hydrated-notification-schema+ [:merge+ ::CreateNotificationHandlerParams+ [:map {:closed true}+ [:template {:optional true} [:maybe ::models.channel/ChannelTemplateUserProvided]][:channel {:optional true} [:maybe ::models.channel/Channel]]- [:recipients {:optional true} [:sequential ::NotificationRecipient]]]]))+ [:recipients {:optional true} [:sequential ::CreateNotificationRecipientParams]]]]+ {:with-id? false}))(mu/defn hydrate-notification :- [:or ::FullyHydratedNotification [:sequential ::FullyHydratedNotification]]"Fully hydrate notifictitons."- [notification-or-notifications]+ [notification-or-notifications :- [:or NotificationWithInlinePayload+ NotificationWithRawHandlers+ [:ref ::FullyHydratedNotification]+ [:sequential ::notification.schema/notification]]](t2/hydrate notification-or-notifications :creator :payload :subscriptions [:handlers :channel :template [:recipients :recipients-detail]]))(mu/defn notifications-for-card :- [:sequential ::FullyHydratedNotification]diff --git a/src/metabase/notification/payload/core.clj b/src/metabase/notification/payload/core.cljindex f4b862a9a4a..7b508963de7 100644--- a/src/metabase/notification/payload/core.clj+++ b/src/metabase/notification/payload/core.clj@@ -1,11 +1,14 @@(ns metabase.notification.payload.core(:require[clojure.walk :as w]+ [malli.util :as mut][metabase.appearance.core :as appearance]+ [metabase.channel.models.channel :as models.channel][metabase.notification.db :as notification.db][metabase.notification.models :as models.notification][metabase.notification.payload.execute :as notification.payload.execute][metabase.notification.payload.temp-storage :as notification.payload.temp-storage]+ [metabase.parameters.schema :as parameters.schema][metabase.premium-features.core :as premium-features][metabase.system.core :as system][metabase.util.malli :as mu]@@ -24,50 +27,74 @@cleanup!cleanable?])+(def ^:private RowTimestamp+ "A `created_at`/`updated_at` column: a `java.time` value on the way out, and -- because clients echo the+ notification they were handed back at us on update -- the ISO string it was encoded as on the way back in."+ [:maybe [:or ms/TemporalInstant ms/TemporalString]])++(def ^:private notification-common-entries+ "Entries every notification map has, whatever its `:payload_type`."+ [[:payload_type (into [:enum] models.notification/notification-types)]+ ;; allow unsaved notification to be sent+ [:id {:optional true} [:maybe ms/PositiveInt]]+ [:creator_id {:optional true} [:maybe ms/PositiveInt]]+ [:active {:optional true} :boolean]+ [:created_at {:optional true} RowTimestamp]+ [:updated_at {:optional true} RowTimestamp]+ [:subscriptions {:optional true} [:sequential ::models.notification/NotificationSubscription]]+ ;; the subscription that triggered this notification+ [:triggering_subscription {:optional true} ::models.notification/NotificationSubscription]+ [:creator {:optional true} [:maybe :metabase.users.schema/user]]+ [:handlers {:optional true}+ [:maybe [:sequential+ [:merge+ ::models.notification/NotificationHandler+ [:map {:closed true}+ [:channel {:optional true} [:maybe ::models.channel/Channel]]+ [:template {:optional true} [:maybe ::models.channel/ChannelTemplate]]+ [:recipients {:optional true} [:sequential ::models.notification/NotificationRecipient]]+ [:attachment_only {:optional true} [:maybe :boolean]]+ [:include_pdf {:optional true} [:maybe :boolean]]]]]]]+ [:payload_id {:optional true} [:maybe ms/PositiveInt]]])++(def ^:private NotificationBase+ "The common notification entries as a closed map, for `[:notification/system-event]` and the other payload types."+ (into [:map {:closed true}] notification-common-entries))+(mr/def ::Notification"Schema for the notification."- ;; TODO: how do we make this schema closed after :merge?- [:merge #_{:closed true}- [:map- [:payload_type (into [:enum] models.notification/notification-types)]- ;; allow unsaved notification to be sent- [:id {:optional true} [:maybe ms/PositiveInt]]- [:active {:optional true} :boolean]- [:created_at {:optional true} :any]- [:updated_at {:optional true} :any]- [:subscriptions {:optional true} [:sequential ::models.notification/NotificationSubscription]]- ;; the subscription that triggered this notification- [:triggering_subscription {:optional true} ::models.notification/NotificationSubscription]]- [:multi {:dispatch :payload_type}- ;; system event is a bit special in that part of the payload comes from the event itself- [:notification/system-event- [:map- [:payload- [:map {:closed true}- ;; TODO: event-info schema for each event type- [:event_topic [:fn #(= "event" (-> % keyword namespace))]]- [:event_info [:maybe :map]]]]]]- [:notification/card- [:map- [:payload {:optional true} ::models.notification/NotificationCard]- [:creator_id ms/PositiveInt]]]- [:notification/dashboard- [:map- [:creator_id ms/PositiveInt]- ;; replacement of pulse- [:dashboard_subscription #_{:optional true}- [:map- [:dashboard_id ms/PositiveInt]- [:parameters {:optional true} [:maybe [:sequential :map]]]- [:dashboard_subscription_dashcards {:optional true}- [:sequential [:map- [:card_id [:maybe ms/PositiveInt]]- [:include_csv {:optional true} [:maybe ms/BooleanValue]]- [:include_xls {:optional true} [:maybe ms/BooleanValue]]- [:format_rows {:optional true} [:maybe ms/BooleanValue]]- [:pivot_results {:optional true} [:maybe ms/BooleanValue]]]]]]]]]- ;; for testing only- [:notification/testing :map]]])+ [:multi {:dispatch :payload_type}+ ;; system event is a bit special in that part of the payload comes from the event itself+ [:notification/system-event+ (mut/merge NotificationBase [:map {:closed true} [:payload ::models.notification/SystemEventPayload]])]+ [:notification/card+ (mut/merge NotificationBase [:map {:closed true}+ [:payload {:optional true} ::models.notification/NotificationCard]+ [:creator_id ms/PositiveInt]])]+ [:notification/dashboard+ (mut/merge NotificationBase+ [:map {:closed true}+ [:creator_id ms/PositiveInt]+ ;; replacement of pulse+ [:dashboard_subscription+ [:map {:closed true}+ [:id {:optional true} [:maybe ms/PositiveInt]]+ [:dashboard_id ms/PositiveInt]+ [:disable_links {:optional true} [:maybe :boolean]]+ [:skip_if_empty {:optional true} :boolean]+ [:parameters {:optional true} [:maybe [:sequential ::parameters.schema/parameter-with-optional-type]]]+ [:dashboard_subscription_dashcards {:optional true}+ [:sequential [:map {:closed true}+ [:card_id [:maybe ms/PositiveInt]]+ [:dashboard_card_id {:optional true} [:maybe ms/PositiveInt]]+ [:include_csv {:optional true} [:maybe ms/BooleanValue]]+ [:include_xls {:optional true} [:maybe ms/BooleanValue]]+ [:format_rows {:optional true} [:maybe ms/BooleanValue]]+ [:pivot_results {:optional true} [:maybe ms/BooleanValue]]]]]]]])]+ ;; for testing only+ [:notification/testing+ (mut/merge NotificationBase [:map {:closed true}+ [:payload {:optional true} [:maybe ::models.notification/SystemEventPayload]]])]])(mr/def ::NotificationPayload"Schema for the notification payload."diff --git a/src/metabase/notification/payload/execute.clj b/src/metabase/notification/payload/execute.cljindex 1eee37a838e..e721b748cc4 100644--- a/src/metabase/notification/payload/execute.clj+++ b/src/metabase/notification/payload/execute.clj@@ -5,10 +5,12 @@[metabase.api.common :as api][metabase.channel.urls :as urls][metabase.dashboards.models.dashboard-card :as dashboard-card]+ [metabase.lib.schema.id :as lib.schema.id][metabase.models.interface :as mi][metabase.models.visualization-settings :as viz-settings][metabase.notification.db :as notification.db][metabase.notification.payload.temp-storage :as notification.temp-storage]+ [metabase.parameters.schema :as parameters.schema][metabase.parameters.shared :as shared.params][metabase.query-processor.card :as qp.card][metabase.query-processor.core :as qp]@@ -19,6 +21,7 @@[metabase.util.log :as log][metabase.util.malli :as mu][metabase.util.malli.registry :as mr]+ [metabase.util.malli.schema :as ms][toucan2.core :as t2]))(defn is-card-empty?@@ -319,6 +322,12 @@[:type [:= :tab-title]]]][::mc/default :map]])+(def ^:private ExecuteDashboardOpts+ [:map {:closed true}+ [:spill-budget {:optional true} notification.temp-storage/ResidentBudget]+ [:only-card-ids {:optional true} [:maybe [:set ms/PositiveInt]]]+ [:attached-card-ids {:optional true} [:maybe [:set ms/PositiveInt]]]])+(mu/defn execute-dashboard :- [:sequential ::Part]"Execute a dashboard and return its parts.@@ -330,10 +339,15 @@subscriptions that never render the other cards).- `:attached-card-ids` cards whose results are exported as file attachments; they run to the attachment row limitwhile the rest get the interactive display limits."- ([dashboard-id user-id parameters]+ ([dashboard-id :- ::lib.schema.id/dashboard+ user-id :- ::lib.schema.id/user+ parameters :- [:maybe ::parameters.schema/parameters]](execute-dashboard dashboard-id user-id parameters nil))- ([dashboard-id user-id parameters {:keys [spill-budget only-card-ids attached-card-ids]- :or {spill-budget (new-spill-budget)}}]+ ([dashboard-id :- ::lib.schema.id/dashboard+ user-id :- ::lib.schema.id/user+ parameters :- [:maybe ::parameters.schema/parameters]+ {:keys [spill-budget only-card-ids attached-card-ids]+ :or {spill-budget (new-spill-budget)}} :- [:maybe ExecuteDashboardOpts]](let [opts {:spill-budget spill-budget:attached-card-ids attached-card-ids}keep-dashcards (fn [dashcards]diff --git a/src/metabase/notification/payload/temp_storage.clj b/src/metabase/notification/payload/temp_storage.cljindex 823ee207ad5..2a68569eff5 100644--- a/src/metabase/notification/payload/temp_storage.clj+++ b/src/metabase/notification/payload/temp_storage.clj@@ -178,11 +178,11 @@[^StreamingTempFileStorage storage ^java.io.Writer w](.write w (.toString storage)))-(def ^:private ResidentBudget+(def ResidentBudget"Shared mutable budget threaded across all of a notification's cards (created once per notification by[[make-resident-budget]]). `:resident` is an atom tracking the cells currently held in memory across cards;the rest are the cell limits used to decide when a query spills to disk. See [[should-spill?]]."- [:map+ [:map {:closed true}[:resident [:fn #(instance? clojure.lang.IAtom %)]][:per-card pos-int?] ; spill a single query once it alone holds this many cells in memory[:resident-cap pos-int?] ; once resident cells exceed this across cards, squeeze remaining queries down to `:floor`@@ -190,11 +190,21 @@(def ^:private NotificationRffOptions"Options controlling how [[notification-rff]] decides when to spill to disk."- [:map+ [:map {:closed true};; shared across a notification's cards so they can't collectively exhaust memory. A standalone query just passes its;; own freshly-made budget.[:budget ResidentBudget]])+(def ^:private FileContext+ "A debugging label for the data [[notification-rff]] is storing: only ever `pr-str`'d into a log line or frozen+ into a spill file's preamble, never read back by key."+ [:map {:closed true}+ [:dashboard_id {:optional true} [:maybe pos-int?]]+ [:card_id {:optional true} [:maybe pos-int?]]+ [:dashcard_id {:optional true} [:maybe pos-int?]]+ [:card-id {:optional true} [:maybe pos-int?]]+ [:context {:optional true} symbol?]])+(defn make-resident-budget"Create a shared [[ResidentBudget]] for one notification. `limits` is a map of `:per-card`/`:resident-cap`/`:floor`cell counts. Created once (e.g. per dashboard) and passed to every card's [[notification-rff]] via `:budget` so that@@ -370,7 +380,8 @@in the temp file preamble and on the resulting StreamingTempFileStorage (for debugging)."([options :- NotificationRffOptions](notification-rff options {}))- ([options :- NotificationRffOptions file-context]+ ([options :- NotificationRffOptions+ file-context :- FileContext](fn rff [metadata];; `row-storage` is OUR side storage for rows (it spills to disk); the query processor's reducing accumulator is;; the `result` map, which we thread through untouched until completion - exactly likediff --git a/src/metabase/notification/schema.clj b/src/metabase/notification/schema.cljindex c7f008e765f..1740655b5d7 100644--- a/src/metabase/notification/schema.clj+++ b/src/metabase/notification/schema.clj@@ -7,15 +7,10 @@(mr/def ::notification"A Notification as selected from the app DB: every column of `:notification`."- [:map {:closed true}- [:id ms/PositiveInt]- [:payload_type [:or :keyword :string]]- [:active :boolean]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]- [:internal_id {:optional true} [:maybe :string]]- [:payload_id [:maybe ms/PositiveInt]]- [:creator_id [:maybe ::lib.schema.id/user]]])+ [:merge+ ::notification.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::notification.update"What an update (or insert) of a Notification accepts: every column of `:notification` except `id`, all optional."@@ -30,14 +25,10 @@(mr/def ::notification-card"A NotificationCard as selected from the app DB: every column of `:notification_card`."- [:map {:closed true}- [:id ms/PositiveInt]- [:card_id [:maybe ::lib.schema.id/card]]- [:send_once :boolean]- [:send_condition [:or :keyword :string]]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]- [:disable_links [:maybe :boolean]]])+ [:merge+ ::notification-card.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::notification-card.update"What an update (or insert) of a NotificationCard accepts: every column of `:notification_card` except `id`, all optional."@@ -51,15 +42,10 @@(mr/def ::notification-handler"A NotificationHandler as selected from the app DB: every column of `:notification_handler`."- [:map {:closed true}- [:id ms/PositiveInt]- [:channel_type [:or :keyword :string]]- [:notification_id ms/PositiveInt]- [:channel_id [:maybe ms/PositiveInt]]- [:template_id [:maybe ms/PositiveInt]]- [:active :boolean]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]])+ [:merge+ ::notification-handler.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::notification-handler.update"What an update (or insert) of a NotificationHandler accepts: every column of `:notification_handler` except `id`, all optional."@@ -74,19 +60,21 @@(mr/def ::notification-recipient.details"The `:details` column of a NotificationRecipient, decoded."- :map)+ [:or+ [:map {:closed true}]+ [:map {:closed true}+ [:value :string]+ [:channel_id {:optional true} [:maybe :string]]]+ [:map {:closed true}+ [:pattern :string]+ [:is_optional {:optional true} :boolean]]])(mr/def ::notification-recipient"A NotificationRecipient as selected from the app DB: every column of `:notification_recipient`."- [:map {:closed true}- [:id ms/PositiveInt]- [:notification_handler_id ms/PositiveInt]- [:type [:or :keyword :string]]- [:user_id [:maybe ::lib.schema.id/user]]- [:permissions_group_id [:maybe ms/PositiveInt]]- [:details [:maybe ::notification-recipient.details]]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]])+ [:merge+ ::notification-recipient.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::notification-recipient.update"What an update (or insert) of a NotificationRecipient accepts: every column of `:notification_recipient` except `id`, all optional."@@ -101,14 +89,10 @@(mr/def ::notification-subscription"A NotificationSubscription as selected from the app DB: every column of `:notification_subscription`."- [:map {:closed true}- [:id ms/PositiveInt]- [:notification_id ms/PositiveInt]- [:type [:or :keyword :string]]- [:event_name [:maybe [:or :keyword :string]]]- [:created_at ms/TemporalInstant]- [:cron_schedule [:maybe :string]]- [:ui_display_type [:maybe [:or :keyword :string]]]])+ [:merge+ ::notification-subscription.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::notification-subscription.update"What an update (or insert) of a NotificationSubscription accepts: every column of `:notification_subscription` except `id`, all optional."diff --git a/src/metabase/notification/send.clj b/src/metabase/notification/send.cljindex 436631e958c..fa046483d3b 100644--- a/src/metabase/notification/send.clj+++ b/src/metabase/notification/send.clj@@ -453,7 +453,7 @@nil)(def ^:private Options- [:map+ [:map {:closed true}[:notification/sync? :boolean]])(def ^:dynamic *default-options*@@ -484,7 +484,8 @@(mu/defn send-notification!"The function to send a notification. Defaults to `notification.send/send-notification-async!`."- [notification & {:keys [] :as options} :- [:maybe Options]]+ [notification :- ::notification.payload/Notification+ & {:keys [] :as options} :- [:maybe Options]](let [options (merge *default-options* options)sync? (:notification/sync? options)];; with-task-run is a no-op if already nested (e.g., from scheduler)diff --git a/src/metabase/oauth_server/schema.clj b/src/metabase/oauth_server/schema.cljindex 95869c72000..3bbc834cd96 100644--- a/src/metabase/oauth_server/schema.clj+++ b/src/metabase/oauth_server/schema.clj@@ -7,16 +7,10 @@(mr/def ::oauth-access-token"A OAuthAccessToken as selected from the app DB: every column of `:oauth_access_token`."- [:map {:closed true}- [:id ms/PositiveInt]- [:token :string]- [:user_id [:maybe ::lib.schema.id/user]]- [:client_id :string]- [:scope [:sequential :string]]- [:expiry :int]- [:resource [:maybe [:or :string [:sequential :string]]]]- [:revoked_at [:maybe ms/TemporalInstant]]- [:created_at ms/TemporalInstant]])+ [:merge+ ::oauth-access-token.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::oauth-access-token.update"What an update (or insert) of a OAuthAccessToken accepts: every column of `:oauth_access_token` except `id`, all optional."@@ -32,19 +26,10 @@(mr/def ::oauth-authorization-code"A OAuthAuthorizationCode as selected from the app DB: every column of `:oauth_authorization_code`."- [:map {:closed true}- [:id ms/PositiveInt]- [:code :string]- [:user_id ::lib.schema.id/user]- [:client_id :string]- [:redirect_uri :string]- [:scope [:sequential :string]]- [:nonce [:maybe :string]]- [:expiry :int]- [:code_challenge [:maybe :string]]- [:code_challenge_method [:maybe [:or :keyword :string]]]- [:resource [:maybe [:or :string [:sequential :string]]]]- [:created_at ms/TemporalInstant]])+ [:merge+ ::oauth-authorization-code.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::oauth-authorization-code.update"What an update (or insert) of a OAuthAuthorizationCode accepts: every column of `:oauth_authorization_code` except `id`, all optional."@@ -63,25 +48,10 @@(mr/def ::oauth-client"A OAuthClient as selected from the app DB: every column of `:oauth_client`."- [:map {:closed true}- [:id ms/PositiveInt]- [:client_id :string]- [:client_secret_hash [:maybe :string]]- [:redirect_uris [:sequential :string]]- [:grant_types [:sequential :string]]- [:response_types [:sequential :string]]- [:scopes [:sequential :string]]- [:token_endpoint_auth_method [:maybe [:or :keyword :string]]]- [:client_name [:maybe :string]]- [:client_uri [:maybe :string]]- [:logo_uri [:maybe :string]]- [:contacts [:maybe [:sequential :string]]]- [:registration_type [:or :keyword :string]]- [:client_type [:or :keyword :string]]- [:application_type [:maybe [:or :keyword :string]]]- [:registration_access_token_hash [:maybe :string]]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]])+ [:merge+ ::oauth-client.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::oauth-client.update"What an update (or insert) of a OAuthClient accepts: every column of `:oauth_client` except `id`, all optional."@@ -106,12 +76,10 @@(mr/def ::oauth-client-event"A OAuthClientEvent as selected from the app DB: every column of `:oauth_client_event`."- [:map {:closed true}- [:id ms/PositiveInt]- [:oauth_client_id [:maybe ms/PositiveInt]]- [:user_id [:maybe ::lib.schema.id/user]]- [:event_type [:or :keyword :string]]- [:created_at ms/TemporalInstant]])+ [:merge+ ::oauth-client-event.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::oauth-client-event.update"What an update (or insert) of a OAuthClientEvent accepts: every column of `:oauth_client_event` except `id`, all optional."@@ -123,16 +91,10 @@(mr/def ::oauth-refresh-token"A OAuthRefreshToken as selected from the app DB: every column of `:oauth_refresh_token`."- [:map {:closed true}- [:id ms/PositiveInt]- [:token :string]- [:user_id [:maybe ::lib.schema.id/user]]- [:client_id :string]- [:scope [:sequential :string]]- [:resource [:maybe [:or :string [:sequential :string]]]]- [:expiry [:maybe :int]]- [:revoked_at [:maybe ms/TemporalInstant]]- [:created_at ms/TemporalInstant]])+ [:merge+ ::oauth-refresh-token.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::oauth-refresh-token.update"What an update (or insert) of a OAuthRefreshToken accepts: every column of `:oauth_refresh_token` except `id`, all optional."diff --git a/src/metabase/osi/ai_context/api.clj b/src/metabase/osi/ai_context/api.cljindex 53043e79993..0b3d5ea5622 100644--- a/src/metabase/osi/ai_context/api.clj+++ b/src/metabase/osi/ai_context/api.clj@@ -7,6 +7,7 @@[metabase.entity-retrieval.core :as entity-retrieval][metabase.osi.db :as osi.db][metabase.osi.models.osi-ai-context :as osi-ai-context]+ [metabase.osi.schema :as osi.schema][metabase.request.core :as request][metabase.util.malli :as mu][metabase.util.malli.schema :as ms]))@@ -17,23 +18,11 @@measures/segments keep their type. A plain question never matches an index doc, so it's rejected."#{"table" "metric" "model" "measure" "segment"})-(def ^:private max-item-len- "Cap on each synonym/example string — these are short phrases or questions, not prose. They become- embedded index docs, so this also keeps a single value under the embedding provider's token limit."- 1000)--(def ^:private max-list-len- "Cap on the synonyms/examples list length — a curated entity needs a handful, not hundreds."- 50)-(def ^:private AiContext"OSI ai_context blob. All fields optional; extra keys tolerated for forward-compat with the OSI spec.String and list lengths are capped so a single curated entity can't bloat the index, its embeddings, orthe agent prompt."- [:map {:closed true}- [:instructions {:optional true} [:maybe [:string {:max entity-retrieval/max-instructions-len}]]]- [:synonyms {:optional true} [:sequential {:max max-list-len} [:string {:max max-item-len}]]]- [:examples {:optional true} [:sequential {:max max-list-len} [:string {:max max-item-len}]]]])+ ::osi.schema/osi-ai-context.ai-context)(def ^:private AiContextInput"Accepted write shape for `ai_context` — the OSI `AIContext` oneOf.diff --git a/src/metabase/osi/models/osi_ai_context.clj b/src/metabase/osi/models/osi_ai_context.cljindex 67d1dba62cf..397e79ce986 100644--- a/src/metabase/osi/models/osi_ai_context.clj+++ b/src/metabase/osi/models/osi_ai_context.clj@@ -19,6 +19,7 @@[metabase.models.interface :as mi][metabase.models.serialization :as serdes][metabase.osi.db :as osi.db]+ [metabase.osi.schema][metabase.util :as u][methodical.core :as methodical][toucan2.core :as t2]))diff --git a/src/metabase/osi/schema.clj b/src/metabase/osi/schema.cljindex 1ee53e5e2ad..e4288a2c8de 100644--- a/src/metabase/osi/schema.clj+++ b/src/metabase/osi/schema.clj@@ -1,21 +1,31 @@(ns metabase.osi.schema"Malli schemas for the osi module."(:require+ [metabase.entity-retrieval.core :as entity-retrieval][metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms]))+(def ^:private max-item-len+ "Cap on each synonym/example string — these are short phrases or questions, not prose. They become+ embedded index docs, so this also keeps a single value under the embedding provider's token limit."+ 1000)++(def ^:private max-list-len+ "Cap on the synonyms/examples list length — a curated entity needs a handful, not hundreds."+ 50)+(mr/def ::osi-ai-context.ai-context"The `:ai_context` column of a OsiAiContext, decoded."- :map)+ [:map {:closed true}+ [:instructions {:optional true} [:maybe [:string {:max entity-retrieval/max-instructions-len}]]]+ [:synonyms {:optional true} [:sequential {:max max-list-len} [:string {:max max-item-len}]]]+ [:examples {:optional true} [:sequential {:max max-list-len} [:string {:max max-item-len}]]]])(mr/def ::osi-ai-context"A OsiAiContext as selected from the app DB: every column of `:osi_ai_context`."- [:map {:closed true}- [:entity_type [:or :keyword :string]]- [:entity_local_id ms/PositiveInt]- [:ai_context ::osi-ai-context.ai-context]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]])+ [:merge+ ::osi-ai-context.update+ [:map {:closed true}]])(mr/def ::osi-ai-context.update"What an update (or insert) of a OsiAiContext accepts: every column of `:osi_ai_context` except `id`, all optional."diff --git a/src/metabase/parameters/chain_filter.clj b/src/metabase/parameters/chain_filter.cljindex f6a103a8742..2e97908baad 100644--- a/src/metabase/parameters/chain_filter.clj+++ b/src/metabase/parameters/chain_filter.clj@@ -70,6 +70,7 @@[metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.lib.schema :as lib.schema]+ [metabase.lib.schema.common :as lib.schema.common][metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.metadata :as lib.schema.metadata][metabase.lib.types.isa :as lib.types.isa]@@ -99,16 +100,30 @@(mr/def ::constraint"Schema for a constraint on a field."- [:map+ [:map {:closed true}[:field-id ::lib.schema.id/field][:op :keyword] ; name of an MBQL filter clause e.g. `:=` or `:starts-with`- [:value :any]- [:options {:optional true} [:maybe map?]]])+ [:value [:or ms/FieldValue [:sequential ms/FieldValue] [:set ms/FieldValue]]]+ [:options {:optional true} [:maybe [:merge+ ::lib.schema.common/options+ [:map {:closed true}+ [:lib/uuid {:optional true} ::lib.schema.common/uuid]]]]]])(mr/def ::constraints"Schema for a list of constraints."[:sequential ::constraint])+(mr/def ::table-field-endpoint+ [:map {:closed true}+ [:table ::lib.schema.id/table]+ [:field ::lib.schema.id/field]])++(mr/def ::join-info+ "Schema for one FK relationship join between a `:lhs` and `:rhs` Table/Field pair."+ [:map {:closed true}+ [:lhs ::table-field-endpoint]+ [:rhs ::table-field-endpoint]])+(def ^:dynamic *enable-reverse-joins*"Whether to chain filter via joins where we must follow relationships in reverse, e.g. child -> parent (e.g.Restaurant -> Category instead of the usual Category -> Restaurant*)@@ -368,7 +383,7 @@two Tables and we generate the appropriate join against the other Table."[query :- ::lib.schema/querysource-table-id :- ::lib.schema.id/table- joins]+ joins :- [:maybe [:sequential ::join-info]]](let [id->field (u/index-by :id (lib.metadata/bulk-metadata query :metadata/column(into #{} (mapcat (juxt #(get-in % [:lhs :field])#(get-in % [:rhs :field])))@@ -578,7 +593,7 @@(mu/defn- cached-field-values[field-id :- ::lib.schema.id/fieldconstraints :- [:maybe ::constraints]- {:keys [limit], :as _options}]+ {:keys [limit], :as _options} :- [:maybe ::options]];; TODO: why don't we remap the human readable values here?(let [{:keys [values] has-more-values? :has_more_values}(if (empty? constraints)@@ -615,7 +630,7 @@results as a sequence of `[value remapped-value]` pairs."[field-id :- ::lib.schema.id/fieldconstraints :- [:maybe ::constraints]- & options]+ & options :- [:* [:or :keyword :boolean ::lib.schema.id/field ms/PositiveInt]]](assert (even? (count options)))(let [{:as options} optionsrelax-fk-requirement? (:relax-fk-requirement? options)@@ -798,7 +813,7 @@[field-id :- ::lib.schema.id/fieldconstraints :- [:maybe ::constraints]query-string :- [:maybe ms/NonBlankString]- & options]+ & options :- [:* [:or :keyword :boolean ::lib.schema.id/field ms/PositiveInt]]](assert (even? (count options)))(let [{:as options} optionsv->human-readable (delay (schema.metadata-queries/human-readable-remapping-map field-id))diff --git a/src/metabase/parameters/core.clj b/src/metabase/parameters/core.cljindex 6b2db634d53..8a4de2c8384 100644--- a/src/metabase/parameters/core.clj+++ b/src/metabase/parameters/core.clj@@ -19,9 +19,5 @@remapped-valuesearch-values-from-field-id][metabase.parameters.schema- normalize-parameter- normalize-parameter-mapping- normalize-parameter-mappings- normalize-parameterstransform-parameter-mappingstransform-parameters])diff --git a/src/metabase/parameters/custom_values.clj b/src/metabase/parameters/custom_values.cljindex 9c981af6ec9..946e410e437 100644--- a/src/metabase/parameters/custom_values.clj+++ b/src/metabase/parameters/custom_values.clj@@ -64,16 +64,15 @@1000)(mr/def ::values-from-card-query.options- [:map- ;; despite this being called "query string" it can actually be any value because it just gets used in an `:=`- ;; filter clause. :eyeroll:- [:query-string {:optional true} :any]+ [:map {:closed true}+ [:query-string {:optional true} [:maybe ms/FieldValue]];; when present, the matching column is added as a second breakout so that each row becomes a;; [value label] pair used for remapping[:label-field {:optional true} [:maybe [:or :mbql.clause/field :mbql.clause/expression]]];; when present, restrict the values to an exact match on the value column (used to fetch the;; remapped label for a single selected value)- [:exact-value {:optional true} :any]])+ [:exact-value {:optional true} [:maybe [:or ms/FieldValue [:sequential ms/FieldValue]]]]+ [:stage-number {:optional true} [:maybe :int]]])(mu/defn- card-query :- [:maybe ::lib.schema/query]"Build the lib query for the value-source Card identified by `card-id`. `query` is the Card's own `:dataset_query`,@@ -162,13 +161,14 @@{:values [[\"Red Medicine\"]]:has_more_values false}"- ([card field-ref]+ ([card :- :metabase.queries.schema/card+ field-ref :- [:or :mbql.clause/field :mbql.clause/expression]](values-from-card card field-ref nil))([card :- :metabase.queries.schema/cardfield-ref :- [:or :mbql.clause/field :mbql.clause/expression]opts :- [:maybe ::values-from-card-query.options]]- (values-from-card* (card-query (:id card) (not-empty (:dataset_query card))) field-ref opts)))+ (values-from-card* (card-query (:id card) (some-> (:dataset_query card) not-empty lib-be/normalize-query)) field-ref opts)))(defn- can-get-card-values?"Whether the prebuilt value-source `query` exposes the `value-field` column."@@ -198,7 +198,9 @@- :values_source_type = nil."[parameter :- ::parameters.schema/resolved-parameterquery-string :- [:maybe ms/NonBlankString]- default-case-thunk :- [:=> [:cat :any] ms/FieldValuesResult]]+ default-case-thunk :- [:=> [:cat :any] [:map {:closed true}+ [:has_more_values :boolean]+ [:values ms/FieldValuesList]]]](case (:values_source_type parameter):static-list (static-list-values parameter query-string):card (let [config (:values_source_config parameter)@@ -206,7 +208,7 @@(when-not (mi/can-read? card)(throw (ex-info "You don't have permissions to do that." {:status-code 403})))(or (when-not (:archived card)- (when-let [query (card-query (:id card) (not-empty (:dataset_query card)))]+ (when-let [query (card-query (:id card) (some-> (:dataset_query card) not-empty lib-be/normalize-query))](when (can-get-card-values? query (:value_field config))(card-values query config query-string))))(default-case-thunk)))@@ -250,11 +252,12 @@"For a card source configured with a `:label_field`, fetch the [value label] pair for a single`value` by querying the card filtered to that exact value. Returns nil when there is no labelfield, the card is unreadable/archived, or no matching row is found."- [{config :values_source_config :as _param} value]+ [{config :values_source_config :as _param} :- ::parameters.schema/parameter+ value :- [:or ms/FieldValue [:sequential ms/FieldValue]]](when-let [label-field (:label_field config)](when-let [card (parameters.db/card (:card_id config))](when (and (not (:archived card)) (mi/can-read? card))- (when-let [query (card-query (:id card) (not-empty (:dataset_query card)))]+ (when-let [query (card-query (:id card) (some-> (:dataset_query card) not-empty lib-be/normalize-query))](when (can-get-card-values? query (:value_field config))(first (:values (values-from-card* query(lib/->mbql5 (:value_field config))@@ -267,7 +270,7 @@`default-case-thunk` is a 0-arity function that returns values list when :values_source_type = nil."[param :- ::parameters.schema/resolved-parameter- value+ value :- [:or ms/FieldValue [:sequential ms/FieldValue]]default-case-thunk :- [:=> [:cat] :any]](case (:values_source_type param):static-list (m/find-first #(and (vector? %) (= (count %) 2) (= (first %) value))diff --git a/src/metabase/parameters/dashboard.clj b/src/metabase/parameters/dashboard.cljindex 007fb60260d..673f3441b9d 100644--- a/src/metabase/parameters/dashboard.clj+++ b/src/metabase/parameters/dashboard.clj@@ -20,7 +20,8 @@1000)(mu/defn- param->fields- [param :- ::parameters.schema/resolved-parameter & {:keys [op-override]}]+ [param :- ::parameters.schema/resolved-parameter+ & {:keys [op-override]} :- [:maybe [:map {:closed true} [:op-override {:optional true} [:maybe :keyword]]]]](let [op (or op-override (params/param-type->op (:type param)))options (or (:options param) (params/param-type->default-options (:type param)))](for [field-id (params/dashboard-param->field-ids param)]@@ -29,8 +30,8 @@:options options})))(mu/defn ^:private chain-filter-constraints :- ::chain-filter/constraints- [dashboard :- :map- constraint-param-key->value :- [:map-of string? any?]]+ [dashboard :- :metabase.dashboards.schema/dashboard+ constraint-param-key->value :- [:map-of :string [:or ms/FieldValue [:sequential ms/FieldValue]]]](vec (for [[param-key value] constraint-param-key->value:let [param (get-in dashboard [:resolved-params param-key])]:when param@@ -76,12 +77,14 @@"C H A I N filters!Used to query for values that populate chained filter dropdowns and text search boxes."- ([dashboard param-key constraint-param-key->value]+ ([dashboard :- :metabase.dashboards.schema/dashboard+ param-key :- ms/NonBlankString+ constraint-param-key->value :- [:map-of :string [:or ms/FieldValue [:sequential ms/FieldValue]]]](chain-filter dashboard param-key constraint-param-key->value nil))- ([dashboard :- :map+ ([dashboard :- :metabase.dashboards.schema/dashboardparam-key :- ms/NonBlankString- constraint-param-key->value :- [:map-of string? any?]+ constraint-param-key->value :- [:map-of :string [:or ms/FieldValue [:sequential ms/FieldValue]]]query :- [:maybe ms/NonBlankString]](let [dashboard (cond-> dashboard(nil? (:resolved-params dashboard)) (t2/hydrate :resolved-params))@@ -130,9 +133,9 @@1000th-earliest rather than the column's max. Extremes merge across a multi-field parameter exactly(the min of mins bounds every field, as does the max of maxes); the distinct count cannot, and isdropped rather than guessed at."- [dashboard :- :map+ [dashboard :- :metabase.dashboards.schema/dashboardparam-key :- ms/NonBlankString- constraint-param-key->value :- [:map-of string? any?]]+ constraint-param-key->value :- [:map-of :string [:or ms/FieldValue [:sequential ms/FieldValue]]]](let [dashboard (cond-> dashboard(nil? (:resolved-params dashboard)) (t2/hydrate :resolved-params))constraints (chain-filter-constraints dashboard constraint-param-key->value)@@ -159,12 +162,14 @@- static-list: user defined values list- card: values is result of running a card- nil: chain-filter"- ([dashboard param-key constraint-param-key->value]+ ([dashboard :- :metabase.dashboards.schema/dashboard+ param-key :- ms/NonBlankString+ constraint-param-key->value :- [:map-of :string [:or ms/FieldValue [:sequential ms/FieldValue]]]](param-values dashboard param-key constraint-param-key->value nil))- ([dashboard :- :map+ ([dashboard :- :metabase.dashboards.schema/dashboardparam-key :- ms/NonBlankString- constraint-param-key->value :- [:map-of string? any?]+ constraint-param-key->value :- [:map-of :string [:or ms/FieldValue [:sequential ms/FieldValue]]]query :- [:maybe ms/NonBlankString]](let [dashboard (t2/hydrate dashboard :resolved-params)param (get (:resolved-params dashboard) param-key)]diff --git a/src/metabase/parameters/field.clj b/src/metabase/parameters/field.cljindex ad352cfbd1d..a7653c8198a 100644--- a/src/metabase/parameters/field.clj+++ b/src/metabase/parameters/field.clj@@ -14,7 +14,8 @@[metabase.util.log :as log][metabase.util.malli :as mu][metabase.util.malli.schema :as ms]- [metabase.warehouse-schema.metadata-from-qp :as warehouse-schema.metadata-from-qp])+ [metabase.warehouse-schema.metadata-from-qp :as warehouse-schema.metadata-from-qp]+ [metabase.warehouse-schema.schema :as warehouse-schema.schema])(:import(java.text NumberFormat)))@@ -53,12 +54,15 @@;; -> ((14 \"Marilyne Mohr\")(36 \"Margot Farrell\")(48 \"Maryam Douglas\"))"- ([field search-field]+ ([field :- ::warehouse-schema.schema/field+ search-field :- ::warehouse-schema.schema/field](search-values field search-field nil nil))- ([field search-field value]+ ([field :- ::warehouse-schema.schema/field+ search-field :- ::warehouse-schema.schema/field+ value :- [:maybe ms/NonBlankString]](search-values field search-field value nil))- ([field- search-field+ ([field :- ::warehouse-schema.schema/field+ search-field :- ::warehouse-schema.schema/fieldvalue :- [:maybe ms/NonBlankString]maybe-limit :- [:maybe ms/PositiveInt]](try@@ -73,7 +77,11 @@(mu/defn field->values :- ms/FieldValuesResult"Fetch FieldValues, if they exist, for a `field` and return them in an appropriate format for public/embeddeduse-cases."- [{has-field-values-type :has_field_values, field-id :id, has_more_values :has_more_values, :as field}]+ [{has-field-values-type :has_field_values, field-id :id, has_more_values :has_more_values, :as field}+ :- [:merge+ ::warehouse-schema.schema/field+ [:map {:closed true}+ [:has_more_values {:optional true} [:maybe :boolean]]]]];; TODO: explain why using remapped fields is restricted to `has_field_values=list`(if-let [remapped-field-id (when (= has-field-values-type :list)(chain-filter/remapped-field-id field-id))]diff --git a/src/metabase/parameters/field/search_values_query.clj b/src/metabase/parameters/field/search_values_query.cljindex 768395aef13..d46b7e6fc6b 100644--- a/src/metabase/parameters/field/search_values_query.clj+++ b/src/metabase/parameters/field/search_values_query.clj@@ -22,7 +22,7 @@search-field :- [:or(ms/InstanceOf :model/Field)::lib.schema.metadata/column]- value+ value :- [:maybe ms/NonBlankString]limit :- pos-int?](if-let [value->human-readable-value (schema.metadata-queries/human-readable-remapping-map (u/the-id field))](let [query-string (some-> value u/lower-case-en)]diff --git a/src/metabase/parameters/params.clj b/src/metabase/parameters/params.cljindex 20f67bcbeec..5a7e917be4b 100644--- a/src/metabase/parameters/params.clj+++ b/src/metabase/parameters/params.clj@@ -13,10 +13,12 @@[clojure.set :as set][malli.error :as me][medley.core :as m]+ [metabase.dashboards.schema][metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.lib.schema :as lib.schema]+ [metabase.lib.schema.common :as lib.schema.common][metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.metadata :as lib.schema.metadata][metabase.lib.schema.parameter :as lib.schema.parameter]@@ -95,7 +97,7 @@(mu/defn param-target->field-id :- [:maybe ::lib.schema.id/field]"Parse a Card parameter `target` form, which looks something like `[:dimension [:field-id 100]]`, and return the FieldID it references (if any)."- [target+ [target :- ::lib.schema.common/possibly-unnormalized-clause;; TODO (Cam 9/25/25) -- `card` should actually be required but I don't have all day to fix broken tests from;; before I schematized this.card :- [:maybe :metabase.queries.schema/card]]@@ -224,11 +226,17 @@"Internal bookkeeping for one parameter `mapping` on a hydrated `dashcard`: the dashcard, the mapping, and theparameter target's Field ID when the target is already field-id-based (nil when it must be resolved from filterablecolumns)."- [:map- [:dashcard :map]+ [:map {:closed true}+ [:dashcard :metabase.dashboards.schema/dashboard-card][:param-mapping ::parameters.schema/parameter-mapping-with-dashcard][:param-target-field-id [:maybe ::lib.schema.id/field]]])+(mr/def ::field-id-context+ "Accumulator threaded through [[field-id-into-context-rf]]."+ [:map {:closed true}+ [:card-id->filterable-columns [:map-of ::lib.schema.id/card [:map-of :int [:sequential ::lib.schema.metadata/column]]]]+ [:param-id->field-ids [:map-of ::lib.schema.parameter/id [:set ::lib.schema.id/field]]]])+(mu/defn- card->filterable-columns-query :- [:maybe ::lib.schema/query]"Build the lib query whose filterable columns we want for `card` at `stage-number`, or nil when `card` has no query."[card :- :metabase.queries.schema/card@@ -329,12 +337,13 @@(or(some-> *field-id-context* deref)empty-field-id-context))- ([ctx]+ ([ctx :- ::field-id-context](when (some-> *field-id-context* deref)(swap! *field-id-context* update :card-id->filterable-columnsmerge (:card-id->filterable-columns ctx)))(:param-id->field-ids ctx))- ([ctx {:keys [param-mapping param-target-field-id] :as param-dashcard-info}]+ ([ctx :- ::field-id-context+ {:keys [param-mapping param-target-field-id] :as param-dashcard-info} :- ::param-dashcard-info](let [param-id (:parameter_id param-mapping)];; Get the field id from the field-clause if it contains it. This is the common case;; for mbql queries.@@ -365,7 +374,7 @@(mu/defn- mapping->param-dashcard-info :- ::param-dashcard-info"Build the `param-dashcard-info` for a parameter `mapping` on `dashcard`, resolving `:param-target-field-id` when thetarget is already field-id-based."- [dashcard :- :map+ [dashcard :- :metabase.dashboards.schema/dashboard-cardmapping :- ::parameters.schema/parameter-mapping-with-dashcard](let [card (find-card-for-mapping dashcard mapping)]{:dashcard dashcard@@ -396,7 +405,7 @@(mu/defn dashcards->param-id->field-ids* :- [:map-of ::lib.schema.parameter/id [:set ::lib.schema.id/field]]"Return map of parameter ids to mapped field ids."- [dashcards]+ [dashcards :- [:sequential :metabase.dashboards.schema/dashboard-card]](let [param-dashcard-infos (into [](mapcat (fn [dashcard](for [mapping (:parameter_mappings dashcard)]@@ -410,7 +419,7 @@(mu/defn- dashcards->param-id->field-ids :- [:map-of ::lib.schema.parameter/id [:set ::lib.schema.id/field]]"Return a map of Parameter ID to the set of Field IDs referenced by parameters in the Cards on the given `dashcards`,or `nil` if none are referenced. `dashcards` must be hydrated with :card."- [dashcards]+ [dashcards :- [:sequential :metabase.dashboards.schema/dashboard-card]](transduce (comp (map :card)(map card->template-tag-id->field-ids))(partial merge-with set/union)@@ -420,7 +429,7 @@(mu/defn dashcards->param-field-ids :- [:set ::lib.schema.id/field]"Return a set of Field IDs referenced by parameters in Cards in the given `dashcards`, or `nil` ifnone are referenced. `dashcards` must be hydrated with :card."- [dashcards]+ [dashcards :- [:sequential :metabase.dashboards.schema/dashboard-card]](into #{} cat (vals (dashcards->param-id->field-ids dashcards))))(mu/defn dashboard-param->field-ids :- [:set ::lib.schema.id/field]@@ -455,7 +464,7 @@Mostly used for determining Fields referenced by Cards for purposes other than processing queries. Filters out`:field` clauses which use names."- [card :- [:maybe :map]]+ [card :- [:maybe :metabase.queries.schema/card]](some-> card :dataset_query not-empty lib-be/normalize-query lib/all-template-tags-id->field-ids))(methodical/defmethod t2/simple-hydrate [:model/Card :param_fields]@@ -470,7 +479,7 @@"Returns a set of all Field IDs referenced by template tags on this card.To get these IDs broken out by the Param ID that references them, use [[card->template-tag-param-id->field-ids]]."- [card :- [:maybe :map]]+ [card :- [:maybe :metabase.queries.schema/card]](some-> card :dataset_query not-empty lib-be/normalize-query lib/all-template-tag-field-ids not-empty))(def ^:private ParamWithMapping@@ -485,10 +494,7 @@"Return map of Dashboard parameter key -> param with resolved `:mappings` (see the `:resolved-params` hydrationin [[metabase.dashboards.models.dashboard]] for an example). Callers that only need the mappings (e.g. the QP) canpass slim dashcards instead of paying for the full hydration."- [dashboard :- [:map- [:parameters [:maybe [:sequential :map]]]- [:dashcards [:maybe [:sequential [:map- [:parameter_mappings [:maybe [:sequential :map]]]]]]]]]+ [dashboard :- :metabase.dashboards.schema/dashboard](let [param-key->mappings (applymerge-with set/union(for [dashcard (:dashcards dashboard)diff --git a/src/metabase/parameters/schema.cljc b/src/metabase/parameters/schema.cljcindex 470f34c8855..ec8ddd57594 100644--- a/src/metabase/parameters/schema.cljc+++ b/src/metabase/parameters/schema.cljc@@ -1,19 +1,19 @@(ns metabase.parameters.schema(:require#?@(:clj- ([metabase.models.interface :as mi]+ ([metabase.lib.core :as lib]+ [metabase.models.interface :as mi][metabase.util.json :as json]))- [metabase.lib.core :as lib][metabase.lib.schema.common :as lib.schema.common][metabase.lib.schema.id :as lib.schema.id]+ [metabase.lib.schema.literal :as lib.schema.literal][metabase.lib.schema.parameter :as lib.schema.parameter][metabase.lib.schema.temporal-bucketing :as lib.schema.temporal-bucketing]- [metabase.util.malli :as mu][metabase.util.malli.registry :as mr]))(mr/def ::human-readable-remapping-map"Schema for the map of actual value -> human-readable value. Cannot be empty."- [:map-of {:min 1} :any [:maybe :string]])+ [:map-of {:min 1} ::lib.schema.literal/param-value [:maybe :string]])(mr/def ::legacy-ref[:multi {:dispatch (fn [x]@@ -107,19 +107,9 @@[:values_source_type {:optional true} [:maybe ::values-source-type]]]);;; TODO (Cam 10/20/25) -- does this need to include `widget-type` as well? Or is that only set for template tags?-(mu/defn normalize-parameter :- ::parameter- "Normalize `parameter` when coming out of the application database or in via an API request."- [parameter]- (lib/normalize ::parameter parameter))-(mr/def ::parameters[:sequential [:ref ::parameter]])-(mu/defn normalize-parameters :- ::parameters- "Normalize `parameters` when coming out of the application database or in via an API request."- [parameters]- (lib/normalize ::parameters parameters))-(mr/def ::parameter-with-optional-type[:merge::parameter@@ -129,13 +119,6 @@(mr/def ::parameters-with-optional-types[:sequential ::parameter-with-optional-type])-(mu/defn normalize-parameters-without-adding-default-types :- ::parameters-with-optional-types- "The same as [[normalize-parameters]], but does not add a default `:type` if it is missing. Needed in some cases- where we infer the type based on the `:widget-type` in the saved parameter declarations inside a Card or Dashboard,- e.g. when running an embedded Card with the Card QP."- [parameters]- (lib/normalize ::parameters-with-optional-types parameters))-#?(:clj(defn json-encoded"Wrap a parameters `schema` so that it also accepts the JSON-encoded string form. The public sharing, embedding and@@ -150,8 +133,8 @@#?(:clj(def transform-parameters"Toucan 2 transform for columns that are sequences of Card/Dashboard parameters."- {:in (comp mi/json-in normalize-parameters)- :out (comp (mi/catch-normalization-exceptions normalize-parameters) mi/json-out-with-keywordization)}))+ {:in (comp mi/json-in #(lib/normalize ::parameters %))+ :out (comp (mi/catch-normalization-exceptions #(lib/normalize ::parameters %)) mi/json-out-with-keywordization)}))(mr/def ::parameter-with-value"A parameter *value* supplied when running a query, as opposed to a stored parameter declaration. These are the keys@@ -165,6 +148,7 @@;; the name of the template tag this value is for, when it can't be inferred from `:target`[:name {:optional true} [:maybe ::lib.schema.common/non-blank-string]][:slug {:optional true} [:maybe ::lib.schema.common/non-blank-string]]+ [:sectionId {:optional true} [:maybe ::lib.schema.common/non-blank-string]][:type {:optional true} [:maybe [:ref ::lib.schema.parameter/type]]][:value {:optional true} [:ref ::lib.schema.parameter/parameter.value]][:default {:optional true} [:ref ::lib.schema.parameter/parameter.value]]@@ -202,25 +186,15 @@[:target ::lib.schema.parameter/target][:card_id {:optional true} [:maybe ::lib.schema.id/card]]])-(mu/defn normalize-parameter-mapping :- ::parameter-mapping- "Normalize `parameter-mappings` when coming out of the application database or in via an API request."- [parameter-mapping]- (lib/normalize ::parameter-mapping parameter-mapping))-(mr/def ::parameter-mappings[:sequential [:ref ::parameter-mapping]])-(mu/defn normalize-parameter-mappings :- [:maybe ::parameter-mappings]- "Normalize `parameter-mappings` when coming out of the application database or in via an API request."- [parameter-mappings :- [:maybe [:sequential :map]]]- (when parameter-mappings- (lib/normalize ::parameter-mappings parameter-mappings)))-#?(:clj(def transform-parameter-mappings"Toucan 2 transform for columns that are sequences of Card/Dashboard parameter mappings."- {:in (comp mi/json-in normalize-parameter-mappings)- :out (comp (mi/catch-normalization-exceptions normalize-parameter-mappings) mi/json-out-with-keywordization)}))+ {:in (comp mi/json-in #(some->> % (lib/normalize ::parameter-mappings)))+ :out (comp (mi/catch-normalization-exceptions #(some->> % (lib/normalize ::parameter-mappings)))+ mi/json-out-with-keywordization)}))(mr/def ::parameter-mapping-with-dashcard"A `::parameter-mapping` resolved against the DashboardCard that carries it. The `:dashcard` is attached server-side@@ -229,7 +203,7 @@[:merge[:ref ::parameter-mapping][:map- [:dashcard {:optional true} :map]]])+ [:dashcard {:optional true} [:ref :metabase.dashboards.schema/dashboard-card]]]])(mr/def ::resolved-parameter"A dashboard parameter with its `:mappings` resolved against the DashboardCards that carry them, as thediff --git a/src/metabase/parameters/shared.cljc b/src/metabase/parameters/shared.cljcindex 5871290207c..3d8610645a3 100644--- a/src/metabase/parameters/shared.cljc+++ b/src/metabase/parameters/shared.cljc@@ -323,7 +323,7 @@(let [tag->param #?(:clj tag->param:cljs (js->clj tag->param))tag->normalized-param (try- (update-vals tag->param parameters.schema/normalize-parameter)+ (update-vals tag->param #(lib/normalize ::parameters.schema/parameter %))(catch #?(:clj Throwable :cljs :default) e(log/warnf "Unable to substitute tags: invalid parameters: %s" (ex-message e))))];; Most of the functions in this pipeline are relating to handling optional blocks in the text which usediff --git a/src/metabase/permissions/db.clj b/src/metabase/permissions/db.cljindex 9d1bb7fa648..b492670afc4 100644--- a/src/metabase/permissions/db.clj+++ b/src/metabase/permissions/db.clj@@ -504,8 +504,8 @@"Insert `revision` into CollectionPermissionGraphRevision."[revision :- [:map {:closed true}[:id {:optional true} ms/PositiveInt]- [:before {:optional true} [:maybe [:or :map :string]]]- [:after {:optional true} [:maybe [:or :map :string]]]+ [:before {:optional true} ::permissions.schema/collection-permission-graph-revision.before]+ [:after {:optional true} ::permissions.schema/collection-permission-graph-revision.after][:user_id {:optional true} [:maybe ::lib.schema.id/user]][:remark {:optional true} [:maybe :string]]]](t2/insert! :model/CollectionPermissionGraphRevision revision))@@ -514,8 +514,8 @@"Insert `revision` into CollectionPermissionGraphRevision and return the new instance."[revision :- [:map {:closed true}[:id {:optional true} ms/PositiveInt]- [:before {:optional true} [:maybe [:or :map :string]]]- [:after {:optional true} [:maybe [:or :map :string]]]+ [:before {:optional true} ::permissions.schema/collection-permission-graph-revision.before]+ [:after {:optional true} ::permissions.schema/collection-permission-graph-revision.after][:user_id {:optional true} [:maybe ::lib.schema.id/user]][:remark {:optional true} [:maybe :string]]]](first (t2/insert-returning-instances! :model/CollectionPermissionGraphRevision revision)))@@ -524,8 +524,8 @@"Insert `revision` into PermissionsRevision and return the new instance."[revision :- [:map {:closed true}[:id {:optional true} ms/PositiveInt]- [:before {:optional true} [:maybe [:or :map :string]]]- [:after {:optional true} [:maybe [:or :map :string]]]+ [:before {:optional true} [:maybe ::permissions.schema/permissions-revision.before]]+ [:after {:optional true} [:maybe ::permissions.schema/permissions-revision.after]][:user_id {:optional true} [:maybe ::lib.schema.id/user]][:remark {:optional true} [:maybe :string]]]](first (t2/insert-returning-instances! :model/PermissionsRevision revision)))@@ -534,8 +534,8 @@"Insert `revision` into ApplicationPermissionsRevision and return the new instance."[revision :- [:map {:closed true}[:id {:optional true} ms/PositiveInt]- [:before {:optional true} [:maybe [:or :map :string]]]- [:after {:optional true} [:maybe [:or :map :string]]]+ [:before {:optional true} [:maybe ::permissions.schema/application-permissions-revision.before]]+ [:after {:optional true} [:maybe ::permissions.schema/application-permissions-revision.after]][:user_id {:optional true} [:maybe ::lib.schema.id/user]][:remark {:optional true} [:maybe :string]]]](first (t2/insert-returning-instances! :model/ApplicationPermissionsRevision revision)))diff --git a/src/metabase/permissions/models/collection/graph.clj b/src/metabase/permissions/models/collection/graph.cljindex 1ac014f4dca..cb9c8d4b411 100644--- a/src/metabase/permissions/models/collection/graph.clj+++ b/src/metabase/permissions/models/collection/graph.clj@@ -256,7 +256,7 @@**Revision Tracking:**All changes are tracked in `CollectionPermissionGraphRevision` for auditing purposes. The revision numberis checked to prevent concurrent modification conflicts (unless `force?` is true)."- ([new-graph]+ ([new-graph :- PermissionsGraph](update-graph! nil new-graph false))([collection-namespace :- [:maybe ms/KeywordOrString]diff --git a/src/metabase/permissions/models/data_permissions.clj b/src/metabase/permissions/models/data_permissions.cljindex 253f9afb79e..e852e7f813e 100644--- a/src/metabase/permissions/models/data_permissions.clj+++ b/src/metabase/permissions/models/data_permissions.clj@@ -18,6 +18,8 @@[metabase.util.malli :as mu][metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms]+ [metabase.warehouse-schema.schema]+ [metabase.warehouses.schema][methodical.core :as methodical][toucan2.core :as t2])(:import@@ -82,7 +84,7 @@~@body))))(mu/defn- with-cluster-lock-fn- [m :- [:map+ [m :- [:map {:closed true}[:db-id ms/PositiveInt][:perm-type :string]]f :- fn?]@@ -582,7 +584,10 @@(mu/defn table-permission-for-user :- ::permissions.schema/data-permission-value"Returns the effective permission value for a given user, permission type, and database ID, and table ID. If the userhas multiple permissions for the given type in different groups, they are coalesced into a single value."- [user-id perm-type database-id table-id]+ [user-id :- [:maybe ::lib.schema.id/user]+ perm-type :- ::permissions.schema/data-permission-type+ database-id :- ::lib.schema.id/database+ table-id :- ms/IntGreaterThanOrEqualToZero](when (not= :model/Table (model-by-perm-type perm-type))(throw (ex-info (tru "Permission type {0} is a database-level permission." perm-type){perm-type (permissions.schema/data-permissions perm-type)})))@@ -615,7 +620,11 @@(mu/defn user-has-permission-for-table? :- :boolean"Returns a Boolean indicating whether the user has the specified permission value for the given database ID and table ID,or a more permissive value."- [user-id perm-type perm-value database-id table-id]+ [user-id :- [:maybe ::lib.schema.id/user]+ perm-type :- ::permissions.schema/data-permission-type+ perm-value :- ::permissions.schema/data-permission-value+ database-id :- ::lib.schema.id/database+ table-id :- ms/IntGreaterThanOrEqualToZero](at-least-as-permissive? perm-type(table-permission-for-user user-id perm-type database-id table-id)perm-value))@@ -641,7 +650,10 @@Schema names are compared with nil and the empty string treated as equivalent, matching how `database-schemas`presents these databases to the API."- [user-id perm-type database-id schema-name :- [:maybe :string]]+ [user-id :- ::lib.schema.id/user+ perm-type :- ::permissions.schema/data-permission-type+ database-id :- ::lib.schema.id/database+ schema-name :- [:maybe :string]](when (not= :model/Table (model-by-perm-type perm-type))(throw (ex-info (tru "Permission type {0} is not a table-level permission." perm-type){perm-type (permissions.schema/data-permissions perm-type)})))@@ -663,7 +675,11 @@(mu/defn user-has-permission-for-schema? :- :boolean"Returns a Boolean indicating whether the user has the specified permission value for the given database ID and schema,or a more permissive value."- [user-id perm-type perm-value database-id schema]+ [user-id :- ::lib.schema.id/user+ perm-type :- ::permissions.schema/data-permission-type+ perm-value :- ::permissions.schema/data-permission-value+ database-id :- ::lib.schema.id/database+ schema :- [:maybe :string]](at-least-as-permissive? perm-type(schema-permission-for-user user-id perm-type database-id schema)perm-value))@@ -687,7 +703,10 @@Deliberately uncached: the only caller is the upload path, which asks about a single schema of a single database.It asks twice, once per permission type, so a cache would save at most one small scoped query per uploads-enableddatabase -- not worth keeping a fifth cache alive for."- [user-id perm-type database-id schema-name]+ [user-id :- ::lib.schema.id/user+ perm-type :- ::permissions.schema/data-permission-type+ database-id :- ::lib.schema.id/database+ schema-name :- [:maybe :string]](when (not= :model/Table (model-by-perm-type perm-type))(throw (ex-info (tru "Permission type {0} is not a table-level permission." perm-type){perm-type (permissions.schema/data-permissions perm-type)})))@@ -710,7 +729,9 @@(mu/defn database-permission-for-user :- ::permissions.schema/data-permission-value"Returns the effective permission value for a given user, permission type, and database ID. If the user hasmultiple permissions for the given type in different groups, they are coalesced into a single value."- [user-id perm-type database-id]+ [user-id :- ::lib.schema.id/user+ perm-type :- ::permissions.schema/data-permission-type+ database-id :- ::lib.schema.id/database](when (not= :model/Database (model-by-perm-type perm-type))(throw (ex-info (tru "Permission type {0} is a table-level permission." perm-type){perm-type (permissions.schema/data-permissions perm-type)})))@@ -722,7 +743,10 @@(mu/defn user-has-permission-for-database? :- :boolean"Returns a Boolean indicating whether the user has the specified permission value for the given database ID and table ID,or a more permissive value."- [user-id perm-type perm-value database-id]+ [user-id :- ::lib.schema.id/user+ perm-type :- ::permissions.schema/data-permission-type+ perm-value :- ::permissions.schema/data-permission-value+ database-id :- ::lib.schema.id/database](at-least-as-permissive? perm-type(database-permission-for-user user-id perm-type database-id)perm-value))@@ -731,7 +755,9 @@"Returns the effective *db-level* permission value for a given user, permission type, and database ID. If the userhas multiple permissions for the given type in different groups, they are coalesced into a single value. Thedb-level permission is the *most* restrictive table-level permission within that database."- [user-id perm-type database-id]+ [user-id :- [:maybe ::lib.schema.id/user]+ perm-type :- ::permissions.schema/data-permission-type+ database-id :- ::lib.schema.id/database](when (not= :model/Table (model-by-perm-type perm-type))(throw (ex-info (tru "Permission type {0} is not a table-level permission." perm-type){perm-type (permissions.schema/data-permissions perm-type)})))@@ -761,10 +787,13 @@Called without a `database-id`, answers the same question across every database at once -- for asking about theuser's access to the instance as a whole. That takes one query, where walking the databases would take one each."- ([user-id perm-type]+ ([user-id :- [:maybe ::lib.schema.id/user]+ perm-type :- ::permissions.schema/data-permission-type](most-permissive-database-permission-for-user user-id perm-type nil))- ([user-id perm-type database-id]+ ([user-id :- [:maybe ::lib.schema.id/user]+ perm-type :- ::permissions.schema/data-permission-type+ database-id :- [:maybe ::lib.schema.id/database]](when (not= :model/Table (model-by-perm-type perm-type))(throw (ex-info (tru "Permission type {0} is not a table-level permission." perm-type){perm-type (permissions.schema/data-permissions perm-type)})))@@ -794,7 +823,10 @@Answered from [[*db-permission-cache*]] — one aggregated query, memoized per request — so callers that invoke thisrepeatedly within one request (e.g. once per snippet in a list) cost at most one query, however many databases theinstance has. Note that permission rows for inactive tables do not count."- [user-id perm-type & {:keys [exclude-db-ids]}]+ [user-id :- ::lib.schema.id/user+ perm-type :- ::permissions.schema/data-permission-type+ & {:keys [exclude-db-ids]} :- [:maybe [:map {:closed true}+ [:exclude-db-ids {:optional true} [:maybe [:sequential ::lib.schema.id/database]]]]]](or (is-superuser? user-id)(and (= perm-type :perms/manage-table-metadata)(is-data-analyst? user-id))@@ -840,7 +872,10 @@(mu/defn table-permission-for-groups :- ::permissions.schema/data-permission-value"Returns the effective permission value provided by a set of *group-ids*, for a provided permission type, databaseID, and table ID."- [group-ids perm-type database-id table-id]+ [group-ids :- [:set ms/PositiveInt]+ perm-type :- ::permissions.schema/data-permission-type+ database-id :- ::lib.schema.id/database+ table-id :- [:maybe ms/IntGreaterThanOrEqualToZero]](when (not= :model/Table (model-by-perm-type perm-type))(throw (ex-info (tru "Permission type {0} is not a table-level permission." perm-type){perm-type (permissions.schema/data-permissions perm-type)})))@@ -855,7 +890,11 @@(mu/defn groups-have-permission-for-table? :- :boolean"Returns a Boolean indicating whether the provided groups grant the specified permission level or higher for the giventable ID, or a more permissive value. (i.e. if a user is in all of these groups, would they have this permission?)"- [group-ids perm-type perm-value database-id table-id]+ [group-ids :- [:set ms/PositiveInt]+ perm-type :- ::permissions.schema/data-permission-type+ perm-value :- ::permissions.schema/data-permission-value+ database-id :- ::lib.schema.id/database+ table-id :- ms/IntGreaterThanOrEqualToZero](at-least-as-permissive? perm-type(table-permission-for-groups group-ids perm-type database-id table-id)perm-value))@@ -877,7 +916,10 @@This is intended to be used for logging and debugging purposes, to see what a user's real permissions are at a glance. Enforcementshould happen via `database-permission-for-user` and `table-permission-for-user`."- [user-id & {:keys [db-id perm-type]}]+ [user-id :- ::lib.schema.id/user+ & {:keys [db-id perm-type]} :- [:maybe [:map {:closed true}+ [:db-id {:optional true} [:maybe ::lib.schema.id/database]]+ [:perm-type {:optional true} [:maybe ::permissions.schema/data-permission-type]]]]](if (is-superuser? user-id)(admin-permission-graph :db-id db-id :perm-type perm-type)(let [data-perms (permissions.db/user-data-permissions user-id db-id perm-type)@@ -959,7 +1001,17 @@(def ^:private TheIdable"An ID, or something with an ID."- [:or pos-int? [:map [:id pos-int?]]])+ [:or pos-int?+ ::permissions.schema/permissions-group+ :metabase.warehouses.schema/database+ :metabase.warehouse-schema.schema/table])++(def ^:private PermsIndex+ "An in-memory index of DataPermissions rows, as built by [[index-database-permissions]]: `{[group-id db-id+ perm-type] [DataPermissions-row ...]}`."+ [:map-of+ [:tuple pos-int? pos-int? ::permissions.schema/data-permission-type]+ [:sequential ::permissions.schema/data-permissions]])(defn- merge-perm-changes"Merges `{:to-delete [...] :to-insert [...]}` maps, deduping as it concatenates: several implication@@ -979,7 +1031,7 @@Returns a map with keys:- :to-delete - sequence of DataPermissions models to delete- :to-insert - sequence of DataPermissions models to insert "- [perms+ [perms :- PermsIndexgroup-or-id :- TheIdabledb-or-id :- TheIdableperm-type :- ::permissions.schema/data-permission-type@@ -1063,9 +1115,7 @@:perm-type (u/qualified-name perm-type)}(set-database-permission! (index-database-permissions [group-id] [db-id])group-or-id db-or-id perm-type value))))- ([perms :- [:map-of- [:tuple pos-int? pos-int? ::permissions.schema/data-permission-type]- [:sequential :any]]+ ([perms :- PermsIndexgroup-or-id :- TheIdabledb-or-id :- TheIdableperm-type :- ::permissions.schema/data-permission-typediff --git a/src/metabase/permissions/models/data_permissions/sql.clj b/src/metabase/permissions/models/data_permissions/sql.cljindex 93c339c1ae7..c35d1de1401 100644--- a/src/metabase/permissions/models/data_permissions/sql.clj+++ b/src/metabase/permissions/models/data_permissions/sql.clj@@ -93,7 +93,7 @@[user-id :- pos-int?perm-type :- :keywordrequired-level :- :keyword- & [most-or-least :- [:maybe [:enum :most :least]]]]+ & [most-or-least] :- [:* [:enum :most :least]]][:exists ^:allow-subquery{:select [1]:from [[:data_permissions :dp]]@@ -109,7 +109,7 @@(def UserInfo"The user-id to use in the visibility query and their superuser status."- [:map+ [:map {:closed true}[:user-id pos-int?][:is-superuser? :boolean][:is-data-analyst? {:optional true} :boolean]])@@ -122,6 +122,19 @@::permissions.schema/data-permission-type[:or ::permissions.schema/data-permission-value [:tuple ::permissions.schema/data-permission-value [:enum :most :least]]]])+(def ^:private ColumnOrExp+ "A HoneySQL column reference (or composite-key vector of columns) to filter on."+ [:or :keyword [:vector :keyword]])++(def ^:private VisibleTableFilterSelectOptions+ [:map {:closed true}+ [:active-only? {:optional true} :boolean]])++(def ^:private VisibleTableFilterWithCteOptions+ [:map {:closed true}+ [:active-only? {:optional true} :boolean]+ [:include-published-via-collection? {:optional true} :boolean]])+(defn- table-source"The plain `metabase_table`, aliased `mt`: permission queries read only sync-owned columns, and the overlay'sderived table is costly here and breaks on MariaDB."@@ -137,7 +150,7 @@[select-column :- [:enum :id :db_id]{:keys [user-id is-superuser? is-data-analyst?]} :- UserInfopermission-mapping :- PermissionMapping- & [{:keys [active-only?] :or {active-only? false}}]]+ & [{:keys [active-only?] :or {active-only? false}}] :- [:* VisibleTableFilterSelectOptions]]^:allow-subquery{:select [(case select-column:id :mt.id@@ -204,11 +217,11 @@published tables in collections the user can read as a source of `:perms/create-queries:query-builder` grants by adding a third UNION ALL branch to the table_permissions CTE.View-data is intentionally not synthesized; it must still come from real data_permissions."- [column-or-exp :- :any+ [column-or-exp :- ColumnOrExp{:keys [user-id is-superuser? is-data-analyst?] :as user-info} :- UserInfopermission-mapping :- PermissionMapping& [{:keys [active-only? include-published-via-collection?]- :or {active-only? false include-published-via-collection? false}}]]+ :or {active-only? false include-published-via-collection? false}}] :- [:* VisibleTableFilterWithCteOptions]];; Superusers see all tables. Data analysts see all tables when checking manage-table-metadata.(if (or is-superuser?(and is-data-analyst?@@ -300,7 +313,7 @@[user-id :- pos-int?perm-type :- :keywordrequired-level :- :keyword- & [most-or-least :- [:maybe [:enum :most :least]]]]+ & [most-or-least] :- [:* [:enum :most :least]]][:exists ^:allow-subquery{:select [1]:from [[:data_permissions :dp]]diff --git a/src/metabase/permissions/models/permissions.clj b/src/metabase/permissions/models/permissions.cljindex b6bba7da1b6..9a6fb00eef1 100644--- a/src/metabase/permissions/models/permissions.clj+++ b/src/metabase/permissions/models/permissions.clj@@ -262,25 +262,25 @@(mu/defn set-has-application-permission-of-type? :- :boolean"Does `permissions-set` grant *full* access to a application permission of type `perm-type`?"- [permissions-set perm-type]+ [permissions-set :- [:maybe [:set :string]]+ perm-type :- [:enum :setting :monitoring :subscription]](set-has-full-permissions? permissions-set (permissions.path/application-perms-path perm-type)))(mu/defn perms-objects-set-for-parent-collection :- [:set perms.u/PathSchema]"Implementation of `perms-objects-set` for models with a `collection_id`, such as Card, Dashboard, or Pulse.This simply returns the `perms-objects-set` of the parent Collection (based on `collection_id`) or for the RootCollection if `collection_id` is `nil`."- ([this read-or-write]- (perms-objects-set-for-parent-collection nil this read-or-write))+ ([collection-id :- [:maybe ms/PositiveInt]+ read-or-write :- [:enum :read :write]]+ (perms-objects-set-for-parent-collection nil collection-id read-or-write))([collection-namespace :- [:maybe ms/KeywordOrString]- this :- [:map- [:collection_id [:maybe ms/PositiveInt]]]+ collection-id :- [:maybe ms/PositiveInt]read-or-write :- [:enum :read :write]];; based on value of read-or-write determine the appropriate function used to calculate the perms path(let [path-fn (case read-or-write:read permissions.path/collection-read-path- :write permissions.path/collection-readwrite-path)- collection-id (:collection_id this)]+ :write permissions.path/collection-readwrite-path)];; now pass that function our collection_id if we have one, or if not, pass it an object representing the Root;; Collection#{(path-fn (or collection-id@@ -309,7 +309,7 @@[instance read-or-write](if (or (= read-or-write :read)(remote-sync/collection-editable? (or (:collection instance) (:collection_id instance))))- (perms-objects-set-for-parent-collection instance read-or-write)+ (perms-objects-set-for-parent-collection (:collection_id instance) read-or-write);; We need to return a dummy permissions string that cannot possibly belong to a user in;; the case where an instance is not syncable due to remote-sync being in ':production' mode#{"___no-remote-sync-access"}))@@ -397,7 +397,7 @@NOTE: This function is meant for internal usage in this namespace only; use one of the other functions like`revoke-data-perms!` elsewhere instead of calling this directly."- [group-or-id :- [:or :map ms/PositiveInt] path :- perms.u/PathSchema]+ [group-or-id :- permissions.path/GroupOrID path :- perms.u/PathSchema](let [group-id (u/the-id group-or-id)paths (conj (perms.u/->v2-path path) path)](when-let [revoked (permissions.db/related-permission-objects group-id path paths)]@@ -445,8 +445,10 @@;;; TODO -- this is a predicate function that returns truthy or falsey, it should end in a `?` -- Cam(mu/defn can-read-audit-helper"Audit instances should only be readable if audit app is enabled."- [model :- :keyword- instance :- :map]+ [model :- [:= :model/Collection]+ instance :- [:map {:closed true}+ [:id ms/PositiveInt]+ [:namespace {:optional true} [:maybe [:or :keyword :string]]]]](if (and (not (premium-features/enable-audit-app?))(case model:model/Collection (audit/is-collection-id-audit? (:id instance))@@ -454,7 +456,7 @@false(case model:model/Collection (mi/current-user-has-full-permissions? :read instance)- (mi/current-user-has-full-permissions? (perms-objects-set-for-parent-collection instance :read)))))+ (mi/current-user-has-full-permissions? (perms-objects-set-for-parent-collection (:collection_id instance) :read)))));;; ---- Collection-based visibility registration ----@@ -561,7 +563,7 @@(mu/defn- check-is-modifiable-collection"Check whether `collection-or-id` refers to a collection that can have permissions modified. Personal collections, theTrash, and descendants of those can't have their permissions modified."- [collection-or-id :- permissions.path/MapOrID]+ [collection-or-id :- permissions.path/CollectionOrID];; skip the whole thing for the root collection, we know it's not a personal collection, trash, or descendant of one;; of them.(when-not (:metabase.collections.models.collection.root/is-root? collection-or-id)@@ -579,13 +581,13 @@(mu/defn revoke-collection-permissions!"Revoke all access for `group-or-id` to a Collection."- [group-or-id :- permissions.path/MapOrID collection-or-id :- permissions.path/MapOrID]+ [group-or-id :- permissions.path/GroupOrID collection-or-id :- permissions.path/CollectionOrID](check-is-modifiable-collection collection-or-id)(delete-related-permissions! group-or-id (permissions.path/collection-readwrite-path collection-or-id)))(mu/defn grant-collection-readwrite-permissions!"Grant full access to a Collection, which means a user can view all Cards in the Collection and add/remove Cards."- [group-or-id :- permissions.path/MapOrID collection-or-id :- permissions.path/MapOrID]+ [group-or-id :- permissions.path/GroupOrID collection-or-id :- permissions.path/CollectionOrID](check-is-modifiable-collection collection-or-id)(when (perms-group/is-tenant-group? group-or-id)(throw (ex-info (tru "Tenant groups cannot have write access to any collections.") {})))@@ -593,7 +595,7 @@(mu/defn grant-collection-read-permissions!"Grant read access to a Collection, which means a user can view all Cards in the Collection."- [group-or-id :- permissions.path/MapOrID collection-or-id :- permissions.path/MapOrID]+ [group-or-id :- permissions.path/GroupOrID collection-or-id :- permissions.path/CollectionOrID](check-is-modifiable-collection collection-or-id)(let [collection (collection-or-id->collection collection-or-id)](when (perms-group/is-tenant-group? group-or-id)diff --git a/src/metabase/permissions/models/permissions_group_membership.clj b/src/metabase/permissions/models/permissions_group_membership.cljindex a99dd8a4642..96c96da4153 100644--- a/src/metabase/permissions/models/permissions_group_membership.clj+++ b/src/metabase/permissions/models/permissions_group_membership.clj@@ -5,6 +5,7 @@[metabase.events.core :as events][metabase.permissions.db :as permissions.db][metabase.permissions.models.permissions-group :as perms-group]+ [metabase.permissions.schema :as permissions.schema][metabase.util :as u][metabase.util.i18n :refer [deferred-tru tru]][metabase.util.malli :as mu]@@ -119,13 +120,11 @@(mu/defn add-users-to-groups!"Creates permission group memberships from aa sequence of maps of users, groups and is-group-manager?."[pgms :- [:sequential- [:map+ [:map {:closed true}[:group [:orpos-int?- [:map [:id pos-int?]]]]- [:user [:or- pos-int?- [:map [:id pos-int?]]]]+ ::permissions.schema/permissions-group]]+ [:user [:or pos-int? :metabase.users.schema/user]][:is-group-manager? {:optional true}:boolean]]]](when (seq pgms)diff --git a/src/metabase/permissions/path.clj b/src/metabase/permissions/path.cljindex 98e973d47ef..139b6ed5942 100644--- a/src/metabase/permissions/path.clj+++ b/src/metabase/permissions/path.clj@@ -1,18 +1,28 @@(ns metabase.permissions.path(:require[clojure.string :as str]+ [metabase.collections.schema :as collections.schema]+ [metabase.permissions.schema :as permissions.schema][metabase.permissions.util :as perms.u][metabase.util :as u][metabase.util.malli :as mu][metabase.util.malli.schema :as ms]))-(def MapOrID- "Schema for a map or an ID (positive integer)."- [:or :map ms/PositiveInt])+(def CollectionOrID+ "Schema for a Collection, its ID, or the `RootCollection` placeholder object."+ [:or+ ms/PositiveInt+ ::collections.schema/collection-or-root])++(def GroupOrID+ "Schema for a PermissionsGroup or its ID."+ [:or+ ms/PositiveInt+ ::permissions.schema/permissions-group])(mu/defn collection-readwrite-path :- perms.u/PathSchema"Return the permissions path for *readwrite* access for a `collection-or-id`."- [collection-or-id :- MapOrID]+ [collection-or-id :- CollectionOrID](if-not (get collection-or-id :metabase.collections.models.collection.root/is-root?)(format "/collection/%d/" (u/the-id collection-or-id))(if-let [collection-namespace (:namespace collection-or-id)]@@ -31,12 +41,12 @@(mu/defn collection-read-path :- perms.u/PathSchema"Return the permissions path for *read* access for a `collection-or-id`."- [collection-or-id :- MapOrID]+ [collection-or-id :- CollectionOrID](str (collection-readwrite-path collection-or-id) "read/"))(mu/defn application-perms-path :- perms.u/PathSchema"Returns the permissions path for *full* access a application permission."- [perm-type]+ [perm-type :- [:enum :setting :monitoring :subscription]](case perm-type:setting"/application/setting/"diff --git a/src/metabase/permissions/schema.clj b/src/metabase/permissions/schema.cljindex 5884c86c294..1b7dd2cc3b4 100644--- a/src/metabase/permissions/schema.clj+++ b/src/metabase/permissions/schema.clj@@ -35,23 +35,27 @@(into [:enum {:error/message "Invalid permission value"}](distinct (mapcat :values (vals data-permissions)))))+(mr/def ::application-permissions-graph.groups+ "Application permissions by group, or the part of them a change touches: group id -> permission type -> level."+ [:map-of ms/PositiveInt [:map {:closed true}+ [:setting {:optional true} [:enum :yes :no]]+ [:monitoring {:optional true} [:enum :yes :no]]+ [:subscription {:optional true} [:enum :yes :no]]]])+(mr/def ::application-permissions-revision.before"The `:before` column of a ApplicationPermissionsRevision, decoded."- :map)+ ::application-permissions-graph.groups)(mr/def ::application-permissions-revision.after"The `:after` column of a ApplicationPermissionsRevision, decoded."- :map)+ ::application-permissions-graph.groups)(mr/def ::application-permissions-revision"A ApplicationPermissionsRevision as selected from the app DB: every column of `:application_permissions_revision`."- [:map {:closed true}- [:id ms/PositiveInt]- [:before [:maybe ::application-permissions-revision.before]]- [:after [:maybe ::application-permissions-revision.after]]- [:user_id ::lib.schema.id/user]- [:created_at ms/TemporalInstant]- [:remark [:maybe :string]]])+ [:merge+ ::application-permissions-revision.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::application-permissions-revision.update"What an update (or insert) of a ApplicationPermissionsRevision accepts: every column of `:application_permissions_revision` except `id`, all optional."@@ -62,23 +66,29 @@[:created_at {:optional true} [:maybe ms/TemporalInstant]][:remark {:optional true} [:maybe :string]]])+(mr/def ::collection-permission-graph.groups+ "Collection permissions by group, or the part of them a change touches: group id -> collection id (or `:root`) -> level."+ [:map-of ms/PositiveInt [:map-of [:or [:= :root] ms/PositiveInt] [:enum :write :read :none]]])+(mr/def ::collection-permission-graph-revision.before"The `:before` column of a CollectionPermissionGraphRevision, decoded."- :map)+ [:or+ [:= ""]+ [:map {:closed true}+ [:revision {:optional true} [:maybe :int]]+ [:namespace {:optional true} [:maybe [:or :keyword :string]]]+ [:groups {:optional true} ::collection-permission-graph.groups]]])(mr/def ::collection-permission-graph-revision.after"The `:after` column of a CollectionPermissionGraphRevision, decoded."- :map)+ [:or [:= ""] ::collection-permission-graph.groups])(mr/def ::collection-permission-graph-revision"A CollectionPermissionGraphRevision as selected from the app DB: every column of `:collection_permission_graph_revision`."- [:map {:closed true}- [:id ms/PositiveInt]- [:before [:maybe ::collection-permission-graph-revision.before]]- [:after [:maybe ::collection-permission-graph-revision.after]]- [:user_id ::lib.schema.id/user]- [:created_at ms/TemporalInstant]- [:remark [:maybe :string]]])+ [:merge+ ::collection-permission-graph-revision.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::collection-permission-graph-revision.update"What an update (or insert) of a CollectionPermissionGraphRevision accepts: every column of `:collection_permission_graph_revision` except `id`, all optional."@@ -91,15 +101,11 @@(mr/def ::data-permissions"A DataPermissions as selected from the app DB: every column of `:data_permissions`."- [:map {:closed true}- [:id ms/PositiveInt]- [:group_id ms/PositiveInt]- [:perm_type [:or :keyword :string]]- [:db_id ::lib.schema.id/database]- [:schema_name [:maybe :string]]- [:table_id [:maybe ::lib.schema.id/table]]- [:perm_value [:or :keyword :string]]- [:unique_perms_helper {:optional true} [:maybe :int]]])+ [:merge+ ::data-permissions.update+ [:map {:closed true}+ [:id ms/PositiveInt]+ [:unique_perms_helper {:optional true} [:maybe :int]]]])(mr/def ::data-permissions.update"What an update (or insert) of a DataPermissions accepts: every column of `:data_permissions` except `id`, all optional."@@ -113,13 +119,10 @@(mr/def ::permissions"A Permissions as selected from the app DB: every column of `:permissions`."- [:map {:closed true}- [:id ms/PositiveInt]- [:object :string]- [:group_id ms/PositiveInt]- [:perm_value [:maybe [:or :keyword :string]]]- [:perm_type [:maybe [:or :keyword :string]]]- [:collection_id [:maybe ::lib.schema.id/collection]]])+ [:merge+ ::permissions.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::permissions.update"What an update (or insert) of a Permissions accepts: every column of `:permissions` except `id`, all optional."@@ -140,14 +143,13 @@[:routing {:optional true} [:map-of ms/PositiveInt :string]]])(mr/def ::permissions-group- "A PermissionsGroup as selected from the app DB: every column of `:permissions_group`."- [:map {:closed true}- [:id ms/PositiveInt]- [:name :string]- [:entity_id :string]- [:magic_group_type [:maybe [:or :keyword :string]]]- [:is_tenant_group :boolean]- [:is_data_app_group :boolean]])+ "A PermissionsGroup as selected from the app DB: every column of `:permissions_group`, plus `:members` some+ callers hydrate onto it."+ [:merge+ ::permissions-group.update+ [:map {:closed true}+ [:id ms/PositiveInt]+ [:members {:optional true} [:sequential [:map {:closed true} [:email :string] [:entity_id :string]]]]]])(mr/def ::permissions-group.member"A member of a PermissionsGroup as `metabase.permissions.db/group-members` selects it: the User columns the group@@ -176,11 +178,10 @@(mr/def ::permissions-group-membership"A PermissionsGroupMembership as selected from the app DB: every column of `:permissions_group_membership`."- [:map {:closed true}- [:id ms/PositiveInt]- [:user_id ::lib.schema.id/user]- [:group_id ms/PositiveInt]- [:is_group_manager :boolean]])+ [:merge+ ::permissions-group-membership.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::permissions-group-membership.update"What an update (or insert) of a PermissionsGroupMembership accepts: every column of `:permissions_group_membership` except `id`, all optional."@@ -189,23 +190,49 @@[:group_id {:optional true} [:maybe ms/PositiveInt]][:is_group_manager {:optional true} [:maybe :boolean]]])+(mr/def ::data-permissions-graph.schemas+ "An API-style data permission on one database: a level for all of it, or schema name -> a level or table id -> level."+ [:or+ :keyword+ [:map-of :string [:or+ :keyword+ [:map-of ms/PositiveInt [:or+ :keyword+ [:map {:closed true}+ [:read {:optional true} :keyword]+ [:query {:optional true} :keyword]]]]]]])++(mr/def ::data-permissions-graph.native-and-schemas+ "An API-style data permission split into its native-query level and its per-schema levels."+ [:map {:closed true}+ [:native {:optional true} [:maybe :keyword]]+ [:schemas {:optional true} ::data-permissions-graph.schemas]])++(mr/def ::data-permissions-graph.groups+ "API-style data permissions by group, or the part of them a change touches: group id -> database id -> permissions."+ [:map-of ms/PositiveInt [:maybe [:map-of ms/PositiveInt [:map {:closed true}+ [:view-data {:optional true} ::data-permissions-graph.schemas]+ [:create-queries {:optional true} ::data-permissions-graph.schemas]+ [:data {:optional true} ::data-permissions-graph.native-and-schemas]+ [:download {:optional true} ::data-permissions-graph.native-and-schemas]+ [:data-model {:optional true} ::data-permissions-graph.native-and-schemas]+ [:details {:optional true} [:enum :yes :no]]+ [:transforms {:optional true} [:enum :yes :no]]]]]])+(mr/def ::permissions-revision.before"The `:before` column of a PermissionsRevision, decoded."- :map)+ ::data-permissions-graph.groups)(mr/def ::permissions-revision.after"The `:after` column of a PermissionsRevision, decoded."- :map)+ ::data-permissions-graph.groups)(mr/def ::permissions-revision"A PermissionsRevision as selected from the app DB: every column of `:permissions_revision`."- [:map {:closed true}- [:id ms/PositiveInt]- [:before [:maybe ::permissions-revision.before]]- [:after [:maybe ::permissions-revision.after]]- [:user_id ::lib.schema.id/user]- [:created_at ms/TemporalInstant]- [:remark [:maybe :string]]])+ [:merge+ ::permissions-revision.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::permissions-revision.update"What an update (or insert) of a PermissionsRevision accepts: every column of `:permissions_revision` except `id`, all optional."diff --git a/src/metabase/permissions_rest/data_permissions/graph.clj b/src/metabase/permissions_rest/data_permissions/graph.cljindex 5ae3e7c4df7..285403f0350 100644--- a/src/metabase/permissions_rest/data_permissions/graph.clj+++ b/src/metabase/permissions_rest/data_permissions/graph.clj@@ -23,6 +23,14 @@(set! *warn-on-reflection* true)+(mr/def ::opts+ [:map {:closed true}+ [:group-id {:optional true} [:maybe pos-int?]]+ [:group-ids {:optional true} [:maybe [:sequential pos-int?]]]+ [:db-id {:optional true} [:maybe pos-int?]]+ [:audit? {:optional true} [:maybe :boolean]]+ [:perm-type {:optional true} [:maybe ::permissions.schema/data-permission-type]]])+;; See also: [[permissions.schema/data-permissions]](def ^:private ->api-keys{:perms/view-data :view-data@@ -70,11 +78,14 @@(when-not (ellide? type value)[(->api-keys type) ((->api-vals type) value)]))+(mr/def ::schema->table-id->perm-value+ [:map-of :string [:map-of nat-int? ::permissions.schema/data-permission-value]])+(mu/defn- api-table-perms"Helper to transform 'leaf' values with table-level schemas in the data permissions graph into an API-style data permissions value.Coalesces permissions at the schema level if all table-level permissions within a schema are identical."[type :- ::permissions.schema/data-permission-type- schema->table-id->api-val]+ schema->table-id->api-val :- ::schema->table-id->perm-value](let [transform-val (fn [perm-val] ((->api-vals type) perm-val))coalesce-or-transform (fn [table-id->perm](let [unique-perms (set (vals table-id->perm))]@@ -259,7 +270,7 @@"Returns a tree representation of all data permissions. Can be optionally filtered by group ID, database ID,and/or permission type. This is intended to power the permissions editor in the admin panel, and should not be usedfor permission enforcement, as it will read much more data than necessary."- [& {:as opts}]+ [& {:as opts} :- [:maybe ::opts]](reduce-into-graph (data-perms-reducible opts) collapse-uniform-view-data))(defn- maybe-hide-tenant-groups [groups]@@ -277,13 +288,7 @@([](api-graph {}))- ([& {:as opts}- :- [:map- [:group-id {:optional true} [:maybe pos-int?]]- [:group-ids {:optional true} [:maybe [:sequential pos-int?]]]- [:db-id {:optional true} [:maybe pos-int?]]- [:audit? {:optional true} [:maybe :boolean]]- [:perm-type {:optional true} [:maybe ::permissions.schema/data-permission-type]]]]+ ([& {:as opts} :- [:maybe ::opts]]{:revision (perms/latest-permissions-revision-id):groups (-> (reduce-into-graph (data-perms-reducible opts)(fn [perm-map]@@ -602,7 +607,9 @@(mu/defn update-data-perms-graph!*"Takes an API-style perms graph and sets the permissions in the database accordingly.Uses bulk operations to minimize database round-trips."- ([graph]+ ([graph :- [:map-of+ ::permissions-rest.schema/group-id+ [:maybe [:map-of ::permissions-rest.schema/id ::permissions-rest.schema/db-perms]]]](let [affected-group-ids (keys graph)affected-db-ids (into #{} (mapcat keys) (vals graph))all-tables (when (seq affected-db-ids)@@ -620,7 +627,12 @@(perms/batch-insert-permissions! to-insert))));; The following arity is provided solely for convenience for tests/REPL usage- ([ks :- [:vector :any] new-value]+ ([ks :- [:vector [:or :int :string :keyword]]+ new-value :- [:or+ ::permissions.schema/data-permission-value+ [:map-of ::permissions.schema/data-permission-type ::permissions.schema/data-permission-value]+ [:ref ::permissions-rest.schema/schemas]+ [:ref ::permissions-rest.schema/db-perms]]](-> (api-graph):groups(assoc-in ks new-value)@@ -647,7 +659,12 @@(reconcile-data-app-permissions! (into #{} (mapcat keys) (vals group-updates)))))));; The following arity is provided solely for convenience for tests/REPL usage- ([ks :- [:vector :any] new-value]+ ([ks :- [:vector [:or :int :string :keyword]]+ new-value :- [:or+ ::permissions.schema/data-permission-value+ [:map-of ::permissions.schema/data-permission-type ::permissions.schema/data-permission-value]+ [:ref ::permissions-rest.schema/schemas]+ [:ref ::permissions-rest.schema/db-perms]]](-> (api-graph)(assoc-in (cons :groups ks) new-value)update-data-perms-graph!)))diff --git a/src/metabase/permissions_rest/schema.clj b/src/metabase/permissions_rest/schema.cljindex 87fc636b91b..ef0ee12cef7 100644--- a/src/metabase/permissions_rest/schema.clj+++ b/src/metabase/permissions_rest/schema.clj@@ -132,8 +132,9 @@(mr/def ::data-permissions-graph"Used to transform, and verify data permissions graph"- [:map- [:groups [:map-of [:ref ::group-id] [:maybe [:ref ::strict-db-graph]]]]])+ [:map {:closed true}+ [:groups [:map-of [:ref ::group-id] [:maybe [:ref ::strict-db-graph]]]]+ [:revision {:optional true} [:maybe ms/Int]]])(mr/def ::attribute-name"a user attribute name"diff --git a/src/metabase/pivot/postprocess.clj b/src/metabase/pivot/postprocess.cljindex efeb97290c8..c20d19a3bfd 100644--- a/src/metabase/pivot/postprocess.clj+++ b/src/metabase/pivot/postprocess.clj@@ -30,14 +30,19 @@;; The 'pivot-grouping' is the giveaway. If you ever see that column, you know you're dealing with raw pivot rows.(mr/def ::pivot-spec- [:map+ [:map {:closed true}[:column-titles [:sequential [:string]]][:pivot-rows [:sequential [:int {:min 0}]]][:pivot-cols [:sequential [:int {:min 0}]]][:pivot-grouping-key {:optional true}[:int {:min 0}]]+ [:pivot-grouping {:optional true}+ [:int {:min 0}]][:pivot-measures {:optional true}- [:sequential [:int {:min 0}]]]])+ [:sequential [:int {:min 0}]]]+ [:show-row-totals {:optional true} [:maybe :boolean]]+ [:show-column-totals {:optional true} [:maybe :boolean]]+ [:column-sort-order {:optional true} [:maybe [:map-of [:maybe nat-int?] [:maybe :keyword]]]]])(def non-pivot-row-group"Pivot query results have a 'pivot-grouping' column. Rows whose pivot-grouping value is 0 are expected results.diff --git a/src/metabase/plugins/impl.clj b/src/metabase/plugins/impl.cljindex 0df73772dad..6fe68bf01a4 100644--- a/src/metabase/plugins/impl.clj+++ b/src/metabase/plugins/impl.clj@@ -97,7 +97,8 @@(defn- plugin-info [^Path jar-path](some-> (slurp-plugin-manifest-from-archive jar-path)- yaml/parse-string))+ yaml/parse-string+ plugins.init/normalize-manifest))(defn- register-plugin-with-info!"Register a plugin using parsed info from its manifest. Returns truthy if registration was successful;@@ -140,7 +141,7 @@path))(defn- load-plugin-manifest! [path]- (some-> (slurp (str path)) yaml/parse-string plugins.init/register-plugin-with-info!))+ (some-> (slurp (str path)) yaml/parse-string plugins.init/normalize-manifest plugins.init/register-plugin-with-info!))(defn- bundled-manifest-paths"Paths (URIs when running from a jar, `Path`s in dev) for every `metabase-plugin.yaml` on the classpath.diff --git a/src/metabase/plugins/initialize.clj b/src/metabase/plugins/initialize.cljindex edef0b81ad5..f05cea99da2 100644--- a/src/metabase/plugins/initialize.clj+++ b/src/metabase/plugins/initialize.clj@@ -10,7 +10,8 @@[metabase.plugins.lazy-loaded-driver :as lazy-loaded-driver][metabase.util :as u][metabase.util.log :as log]- [metabase.util.malli :as mu])+ [metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr])(:import(java.util.concurrent.locks ReentrantLock)))@@ -178,14 +179,103 @@(defn- info-registered? [{{plugin-name :name} :info}](registered? plugin-name))+(mr/def ::init-step+ "One step under `init:` in a plugin manifest."+ [:multi {:dispatch :step}+ ["load-namespace" [:map {:closed true} [:step [:= "load-namespace"]] [:namespace :string]]]+ ["register-jdbc-driver" [:map {:closed true} [:step [:= "register-jdbc-driver"]] [:class :string]]]])++(mr/def ::dependency+ "One entry under `dependencies:` in a plugin manifest."+ [:or+ [:map {:closed true} [:class :string] [:message {:optional true} :string]]+ [:map {:closed true} [:plugin :string]]+ [:map {:closed true} [:env-var :string]]])++(mr/def ::connection-property+ "One entry under `driver: connection-properties:` in a plugin manifest: a preset name, a full property map, a+ `merge:` override list, or a `group:` of properties."+ [:or+ :string+ [:map {:closed true}+ [:group [:map {:closed true}+ [:container-style {:optional true} [:sequential :string]]+ [:fields [:sequential [:ref ::connection-property]]]]]]+ [:map {:closed true}+ [:merge [:sequential [:ref ::connection-property]]]]+ [:map {:closed true}+ [:name {:optional true} :string]+ [:display-name {:optional true} :string]+ [:helper-text {:optional true} :string]+ [:description {:optional true} :string]+ [:type {:optional true} :string]+ [:secret-kind {:optional true} :string]+ [:required {:optional true} :boolean]+ [:default {:optional true} [:or :string :int :boolean]]+ [:placeholder {:optional true} [:or :string :int :boolean]]+ [:treat-before-posting {:optional true} :string]+ [:visible-if {:optional true} [:map-of :string [:or :string :boolean [:sequential :string]]]]]])++(mr/def ::driver-info+ "The `driver:` section of a plugin manifest."+ [:map {:closed true}+ [:name :string]+ [:display-name {:optional true} :string]+ [:lazy-load {:optional true} :boolean]+ [:abstract {:optional true} :boolean]+ [:parent {:optional true} [:or :string [:sequential :string]]]+ [:connection-properties {:optional true} [:sequential ::connection-property]]+ [:connection-properties-include-tunnel-config {:optional true} :boolean]])++(mr/def ::extra-info+ "The `extra:` section of a plugin manifest."+ [:map {:closed true}+ [:db-routing-info {:optional true} [:map {:closed true} [:text :string]]]])++(mr/def ::manifest+ "A parsed `metabase-plugin.yaml` manifest, as handed to [[register-plugin-with-info!]]."+ [:map {:closed true}+ [:metabase-plugin-api-version {:optional true} :int]+ [:info [:map {:closed true}+ [:name :string]+ [:version :string]+ [:description {:optional true} :string]]]+ [:driver {:optional true} [:maybe [:or ::driver-info [:sequential ::driver-info]]]]+ [:init {:optional true} [:sequential ::init-step]]+ [:dependencies {:optional true} [:sequential ::dependency]]+ [:extra {:optional true} ::extra-info]+ [:superseded-by {:optional true} :string]+ [:add-to-classpath! {:optional true} ifn?]])++(defn- normalize-connection-property+ "Stringifies a YAML-parsed `:visible-if` map's keys (clj-yaml keywordizes them), matching the string keys+ `driver.u/connection-props-server->client` writes for its own generated properties."+ [prop]+ (cond+ (map? (:group prop)) (update-in prop [:group :fields] (partial mapv normalize-connection-property))+ (sequential? (:merge prop)) (update prop :merge (partial mapv normalize-connection-property))+ (and (map? prop) (map? (:visible-if prop))) (update prop :visible-if update-keys u/qualified-name)+ :else prop))++(defn- normalize-driver-info [driver-info]+ (cond-> driver-info+ (some? (:connection-properties driver-info))+ (update :connection-properties (partial mapv normalize-connection-property))))++(defn normalize-manifest+ "Normalizes a freshly YAML-parsed plugin manifest so it matches [[::manifest]] before it is registered."+ [manifest]+ (cond-> manifest+ (some? (:driver manifest))+ (update :driver (fn [driver-or-drivers]+ (if (sequential? driver-or-drivers)+ (mapv normalize-driver-info driver-or-drivers)+ (normalize-driver-info driver-or-drivers))))))+(mu/defn register-plugin-with-info!"Register a plugin using parsed info from its manifest. Returns truthy if the plugin was successfully registered;falsey otherwise."- [info :- [:map- [:metabase-plugin-api-version {:optional true} :int]- [:info [:map- [:name :string]- [:version :string]]]]]+ [info :- ::manifest](validate-plugin-api-version! info)(or (info-registered? info)(dodiff --git a/src/metabase/premium_features/schema.clj b/src/metabase/premium_features/schema.cljindex 67191cde34c..b77047dba7d 100644--- a/src/metabase/premium_features/schema.clj+++ b/src/metabase/premium_features/schema.clj@@ -6,10 +6,9 @@(mr/def ::premium-features-cache"A PremiumFeaturesCache as selected from the app DB: every column of `:premium_features_token_cache`."- [:map {:closed true}- [:token_hash :string]- [:token_status_hash :string]- [:updated_at ms/TemporalInstant]])+ [:merge+ ::premium-features-cache.update+ [:map {:closed true}]])(mr/def ::premium-features-cache.update"What an update (or insert) of a PremiumFeaturesCache accepts: every column of `:premium_features_token_cache` except `id`, all optional."diff --git a/src/metabase/premium_features/token_check.clj b/src/metabase/premium_features/token_check.cljindex a0cfa5e11aa..b20b554ac21 100644--- a/src/metabase/premium_features/token_check.clj+++ b/src/metabase/premium_features/token_check.clj@@ -47,11 +47,6 @@"Similar to RemoteCheckedToken, but starts with 'airgap_'."#"airgap_.+")-(def ^:private TokenStr- [:or- [:re RemoteCheckedToken]- [:re AirgapToken]])-(def ^String token-check-url"Base URL to use for token checks. Hardcoded by default but for development purposes you can use a local server.Specify the env var `METASTORE_DEV_SERVER_URL`. If no server is defined, it uses the staging token check url."@@ -163,9 +158,14 @@(when (seq token)(format "%s/api/%s/v2/status" base-url token)))+(def ^:private Meters+ "The `:meters` of a token status response, keyed by the meter names the license server defines, each meter's fields+ also being the license server's."+ [:map {:closed false, ::mr/deliberately-open true, :description "license-server meters"}])+(def TokenStatus"Schema for a response from the token status API."- [:map+ [:map {:closed true}[:valid :boolean][:status [:string {:min 1}]][:error-details {:optional true} [:maybe [:string {:min 1}]]]@@ -176,10 +176,13 @@[:valid-thru {:optional true} [:string {:min 1}]][:max-users {:optional true} pos-int?][:company {:optional true} [:string {:min 1}]]- [:store-users {:optional true} [:maybe [:sequential [:map+ [:store-users {:optional true} [:maybe [:sequential [:map {:closed true}[:email :string]]]]]- [:meters {:optional true} :map]- [:quotas {:optional true} [:sequential [:map]]]])+ [:meters {:optional true} Meters]+ [:quotas {:optional true} [:sequential [:map {:closed true}+ [:hosting-feature {:optional true} :string]+ [:soft-limit {:optional true} number?]+ [:usage {:optional true} number?]]]]])(defn- http-fetch[base-url token site-uuid]@@ -268,7 +271,7 @@(mu/defn- decode-token* :- TokenStatus"Decode a token. If you get a positive response about the token, even if it is not valid, return that. Errors willbe caught further up with appropriate fall backs, retry strategies, and grace periods for features."- [token :- TokenStr]+ [token :- :string];; NB that we fetch any settings from this thread, not inside on of the futures in the inner fetch calls. We;; will have taken a lock to call through to here, and could create a deadlock with the future's thread. See;; https://github.com/metabase/metabase/pull/38029/diff --git a/src/metabase/public_sharing_rest/api.clj b/src/metabase/public_sharing_rest/api.cljindex 58d84f1e611..a926f84c77c 100644--- a/src/metabase/public_sharing_rest/api.clj+++ b/src/metabase/public_sharing_rest/api.clj@@ -14,7 +14,10 @@[metabase.events.core :as events][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]+ [metabase.lib.schema.constraints :as lib.schema.constraints]+ [metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.info :as lib.schema.info]+ [metabase.lib.schema.middleware-options :as lib.schema.middleware-options][metabase.lib.schema.parameter :as lib.schema.parameter][metabase.models.interface :as mi][metabase.parameters.dashboard :as parameters.dashboard]@@ -24,6 +27,7 @@[metabase.public-sharing.core :as public-sharing][metabase.public-sharing.validation :as public-sharing.validation][metabase.queries.core :as queries]+ [metabase.queries.schema :as queries.schema][metabase.query-processor.card :as qp.card][metabase.query-processor.dashboard :as qp.dashboard][metabase.query-processor.error-type :as qp.error-type]@@ -183,12 +187,21 @@"Run the query for pre-loaded Card `card` with `parameters` and other query options (e.g. `:constraints`).Callers are responsible for resolving `card` themselves with a fresh `public-sharing-rest.db/active-card` call (each endpoint shouldselect the Card exactly once and thread the loaded entity here). Returns a `StreamingResponse`."- [card- export-format- parameters+ [card :- ::queries.schema/card+ export-format :- [:maybe :keyword]+ parameters :- [:maybe ::parameters.schema/api.parameter-values]& {:keys [qp]:or {qp qp.card/process-query-for-card-default-qp}- :as options}]+ :as options} :- [:maybe+ [:map {:closed true}+ [:qp {:optional true} [:maybe ifn?]]+ [:constraints {:optional true} [:maybe ::lib.schema.constraints/constraints]]+ [:context {:optional true} [:maybe ::lib.schema.info/context]]+ [:dashboard-id {:optional true} [:maybe ::lib.schema.id/dashboard]]+ [:dashcard {:optional true} [:maybe ::dashboards.schema/dashboard-card]]+ [:middleware {:optional true} [:maybe ::lib.schema.middleware-options/middleware-options]]+ [:ignore-cache {:optional true} [:maybe :boolean]]+ [:card-transform {:optional true} [:maybe ifn?]]]]];; run this query with full superuser perms;;;; we actually need to bind the current user perms here twice, once so `card-api` will have the full perms when it@@ -284,7 +297,8 @@"Return the public Dashboard with the given `dashboard-id`, removing all columns that should not be visible tothe general public. Throws a 404 if the Dashboard doesn't exist. With `:enable-embedding? true`, additionallyrequires embedding to be enabled."- [dashboard-id & {:as options}]+ [dashboard-id :- [:maybe ::lib.schema.id/dashboard]+ & {:as options} :- [:maybe [:map {:closed true} [:enable-embedding? {:optional true} [:maybe :boolean]]]]](binding [params/*ignore-current-user-perms-and-return-all-field-values* trueparams/*field-id-context* (atom params/empty-field-id-context)](-> (api/check-404 (public-sharing-rest.db/public-dashboard dashboard-id options))diff --git a/src/metabase/pulse/api/pulse.clj b/src/metabase/pulse/api/pulse.cljindex eddd6248eb1..12e88cb586a 100644--- a/src/metabase/pulse/api/pulse.clj+++ b/src/metabase/pulse/api/pulse.clj@@ -99,7 +99,7 @@(t2/with-transaction [_conn];; Adding a new pulse at `collection_position` could cause other pulses in this collection to change position,;; check that and fix it if needed- (api/maybe-reconcile-collection-position! pulse-data)+ (api/maybe-reconcile-collection-position! (select-keys pulse-data [:collection_id :collection_position]));; ok, now create the Pulse(let [pulse (api/check-500(models.pulse/create-pulse! (map models.pulse/card->ref cards) channels pulse-data))]@@ -286,7 +286,7 @@(t2/with-transaction [_conn];; If the collection or position changed with this update, we might need to fixup the old and/or new collection,;; depending on what changed.- (api/maybe-reconcile-collection-position! pulse-before-update pulse-updates)+ (api/maybe-reconcile-collection-position! (select-keys pulse-before-update [:collection_id :collection_position]) (select-keys pulse-updates [:collection_id :collection_position]));; ok, now update the Pulse(models.pulse/update-pulse!(assoc (select-keys pulse-updates [:name :cards :channels :skip_if_empty :collection_id :collection_positiondiff --git a/src/metabase/pulse/models/pulse.clj b/src/metabase/pulse/models/pulse.cljindex 22b10120a64..448bb350d92 100644--- a/src/metabase/pulse/models/pulse.clj+++ b/src/metabase/pulse/models/pulse.clj@@ -23,11 +23,14 @@[metabase.api.common :as api][metabase.collections.models.collection :as collection][metabase.events.core :as events]+ [metabase.lib.core :as lib]+ [metabase.lib.schema.id :as lib.schema.id][metabase.models.interface :as mi][metabase.parameters.schema :as parameters.schema][metabase.permissions.core :as perms][metabase.pulse.db :as pulse.db][metabase.pulse.models.pulse-channel :as pulse-channel]+ [metabase.pulse.schema :as pulse.schema][metabase.util :as u][metabase.util.i18n :refer [deferred-tru tru]][metabase.util.malli :as mu]@@ -47,8 +50,14 @@(derive :hook/entity-id)(derive ::mi/read-policy.full-perms-for-perms-set))+(def ^:private transform-pulse-parameters+ "Toucan 2 transform for a Pulse's `:parameters`, normalized like a Dashboard's but with each `:type` optional."+ {:in (comp mi/json-in #(lib/normalize ::parameters.schema/parameters-with-optional-types %))+ :out (comp (mi/catch-normalization-exceptions #(lib/normalize ::parameters.schema/parameters-with-optional-types %))+ mi/json-out-with-keywordization)})+(t2/deftransforms :model/Pulse- {:parameters mi/transform-json})+ {:parameters transform-pulse-parameters})(defn- assert-valid-parameters [{:keys [parameters]}](when-not (mr/validate [:maybe@@ -137,7 +146,7 @@[notification read-or-write](if (is-alert? notification)(mi/perms-objects-set (alert->card notification) read-or-write)- (perms/perms-objects-set-for-parent-collection notification read-or-write)))+ (perms/perms-objects-set-for-parent-collection (:collection_id notification) read-or-write)))(defn- current-user-is-creator?[notification]@@ -237,8 +246,55 @@"dashboard_id" "parameter_mappings"]))))(def CoercibleToCardRef- "Schema for functions accepting either a `HybridPulseCard`, `CardRef`, or `CardBase`."- [:or HybridPulseCard CardRef CardBase])+ "Schema for functions accepting either a `HybridPulseCard`, `CardRef`, `CardBase`, or a full Card row+ (as `card->ref` accepts \"a card or id\")."+ [:or HybridPulseCard CardRef CardBase :metabase.queries.schema/card])++(def ^:private RecipientInput+ "One entry of `PulseChannelInput`'s `:recipients`: either a Metabase User (by `:id`) or a raw email address."+ [:map {:closed true}+ [:id {:optional true} ms/PositiveInt]+ [:email {:optional true} :string]])++(def ^:private PulseChannelInput+ "A PulseChannel as [[update-notification-channels!]]/[[create-pulse!]] accept it: a subset of `::pulse-channel`+ columns (`:id`/`:pulse_id` filled in server-side when absent) plus `:recipients`."+ [:map {:closed true}+ [:id {:optional true} ms/PositiveInt]+ [:channel_type {:optional true} [:or :keyword :string]]+ [:channel_id {:optional true} [:maybe ms/PositiveInt]]+ [:details {:optional true} ::pulse.schema/pulse-channel.details]+ [:enabled {:optional true} :boolean]+ [:pulse_id {:optional true} ::lib.schema.id/pulse]+ [:entity_id {:optional true} [:maybe :string]]+ [:recipients {:optional true} [:maybe [:sequential [:or RecipientInput :metabase.users.schema/user]]]]+ [:schedule_type {:optional true} [:or :keyword :string]]+ [:schedule_day {:optional true} [:maybe :string]]+ [:schedule_hour {:optional true} [:maybe :int]]+ [:schedule_frame {:optional true} [:maybe [:or :keyword :string]]]+ [:created_at {:optional true} ms/TemporalInstant]+ [:updated_at {:optional true} ms/TemporalInstant]])++(def ^:private UpdateNotificationInput+ "What [[update-notification!]] accepts: the keys of a Pulse/Alert to update, all optional except `:id`."+ [:map {:closed true}+ [:id ms/PositiveInt]+ [:name {:optional true} ms/NonBlankString]+ [:alert_condition {:optional true} AlertConditions]+ [:alert_above_goal {:optional true} boolean?]+ [:alert_first_only {:optional true} boolean?]+ [:skip_if_empty {:optional true} boolean?]+ [:collection_id {:optional true} [:maybe ms/PositiveInt]]+ [:collection_position {:optional true} [:maybe ms/PositiveInt]]+ [:cards {:optional true} [:sequential CoercibleToCardRef]]+ [:channels {:optional true} [:sequential PulseChannelInput]]+ [:archived {:optional true} boolean?]+ [:parameters {:optional true} [:maybe [:sequential ::pulse.schema/pulse.parameter]]]])++(def ^:private NotificationOrId+ "A Pulse/Alert id, a full Pulse row, or [[update-notification!]]'s own argument map — the shapes+ [[u/the-id]] is called with below."+ [:or ms/PositiveInt ::pulse.schema/pulse UpdateNotificationInput]);;; --------------------------------------------------- Hydration ----------------------------------------------------@@ -256,7 +312,7 @@false)(mu/defn- cards* :- [:sequential HybridPulseCard]- [pulse-ids]+ [pulse-ids :- [:sequential ms/PositiveInt]](pulse.db/pulse-cards-for-pulses pulse-ids *allow-hydrate-archived-cards*))(methodical/defmethod t2/batched-hydrate [:model/Pulse :cards]@@ -295,7 +351,7 @@(mu/defn retrieve-pulse :- [:maybe (ms/InstanceOf :model/Pulse)]"Fetch a single *Pulse*, and hydrate it with a set of 'standard' hydrations; remove Alert columns, since this is a*Pulse* and they will all be unset."- [pulse-or-id]+ [pulse-or-id :- NotificationOrId](some-> (pulse.db/pulse (u/the-id pulse-or-id))hydrate-notificationnotification->pulse))@@ -303,7 +359,8 @@(mu/defn retrieve-notification :- [:maybe (ms/InstanceOf :model/Pulse)]"Fetch an Alert or Pulse, and do the 'standard' hydrations, adding `:channels` with `:recipients`, `:creator`, and`:cards`."- [notification-or-id & additional-conditions]+ [notification-or-id :- NotificationOrId+ & additional-conditions :- [:* [:or :keyword :boolean :nil]]]{:pre [(even? (count additional-conditions))]}(let [id (u/the-id notification-or-id)opts (apply hash-map additional-conditions)]@@ -326,18 +383,23 @@(mu/defn retrieve-alert :- [:maybe (ms/InstanceOf :model/Pulse)]"Fetch a single Alert by its `id` value, do the standard hydrations, and put it in the standard `Alert` format."- [alert-or-id]+ [alert-or-id :- NotificationOrId](some-> (pulse.db/alert (u/the-id alert-or-id))hydrate-notificationnotification->alert))+(def ^:private RetrieveAlertsOpts+ [:map {:closed true}+ [:archived? {:optional true} [:maybe :boolean]]+ [:user-id {:optional true} [:maybe ms/PositiveInt]]])+(mu/defn retrieve-alerts :- [:sequential (ms/InstanceOf :model/Pulse)]"Fetch all Alerts."([](retrieve-alerts nil))([{:keys [archived? user-id]- :or {archived? false}}]+ :or {archived? false}} :- [:maybe RetrieveAlertsOpts]](assert boolean? archived?)(for [alert (hydrate-notifications (pulse.db/alerts archived? user-id)):let [alert (notification->alert alert)]@@ -347,11 +409,17 @@:when (:card alert)]alert)))+(def ^:private RetrievePulsesOpts+ [:map {:closed true}+ [:archived? {:optional true} [:maybe :boolean]]+ [:dashboard-id {:optional true} [:maybe ms/PositiveInt]]+ [:user-id {:optional true} [:maybe ms/PositiveInt]]])+(mu/defn retrieve-pulses :- [:sequential (ms/InstanceOf :model/Pulse)]"Fetch all `Pulses`. When `user-id` is included, only fetches `Pulses` for which the provided user is the creatoror a recipient."[{:keys [archived? dashboard-id user-id]- :or {archived? false}}]+ :or {archived? false}} :- RetrievePulsesOpts](for [pulse (pulse.db/pulses archived? dashboard-id user-id)](-> pulse(dissoc :lower-name)@@ -454,7 +522,7 @@(mu/defn retrieve-user-alerts-for-card"Find all alerts for `card-id` that `user-id` is set to receive"[{:keys [archived? card-id user-id]- :or {archived? false}} :- [:map+ :or {archived? false}} :- [:map {:closed true}[:card-id pos-int?][:user-id pos-int?][:archived? {:optional true} boolean?]]]@@ -465,7 +533,7 @@(mu/defn retrieve-alerts-for-cards"Find all alerts for `card-ids`, used for admin users"[{:keys [archived? card-ids]- :or {archived? false}} :- [:map+ :or {archived? false}} :- [:map {:closed true}[:card-ids [:maybe [:or[:sequential pos-int?][:set pos-int?]]]]@@ -476,7 +544,7 @@(mu/defn card->ref :- CardRef"Create a card reference from a card or id"- [card :- :map]+ [card :- CoercibleToCardRef]{:id (u/the-id card):include_csv (get card :include_csv false):include_xls (get card :include_xls false)@@ -493,7 +561,8 @@* If a Card ID in `card-refs` has no corresponding existing `PulseCard` object, one will be created.* If an existing `PulseCard` has no corresponding ID in CARD-IDs, it will be deleted.* All cards will be updated with a `position` according to their place in the collection of `card-ids`"- [notification-or-id card-refs :- [:maybe [:sequential CardRef]]]+ [notification-or-id :- NotificationOrId+ card-refs :- [:maybe [:sequential CardRef]]];; first off, just delete any cards associated with this pulse (we add them again below)(pulse.db/delete-pulse-cards-for-pulse! (u/the-id notification-or-id));; now just insert all of the cards that were given to us@@ -519,7 +588,8 @@* If an existing `PulseChannel` has no corresponding entry in `channels`, it will be deleted.* All previously existing channels will be updated with their most recent information."- [notification-or-id channels :- [:sequential :map]]+ [notification-or-id :- NotificationOrId+ channels :- [:sequential PulseChannelInput]](let [existing-channels (pulse.db/pulse-channels-for-pulse (u/the-id notification-or-id))channels (map-indexed(fn [idx channel]@@ -553,7 +623,9 @@(mu/defn- create-notification-and-add-cards-and-channels!"Create a new Pulse/Alert with the properties specified in `notification`; add the `card-refs` to the Notification andadd the Notification to `channels`. Returns the `id` of the newly created Notification."- [notification card-refs :- [:maybe [:sequential CardRef]] channels]+ [notification :- ::pulse.schema/pulse.update+ card-refs :- [:maybe [:sequential CardRef]]+ channels :- [:sequential PulseChannelInput]](t2/with-transaction [_conn](let [notification (pulse.db/insert-pulse! notification)](update-notification-cards! notification card-refs)@@ -565,16 +637,16 @@PulseCards, PulseChannels, and PulseChannelRecipients.Returns the newly created Pulse, or throws an Exception."- [cards :- [:sequential [:map-of :keyword :any]]- channels :- [:sequential [:map-of :keyword :any]]- kvs :- [:map+ [cards :- [:sequential CardRef]+ channels :- [:sequential PulseChannelInput]+ kvs :- [:map {:closed true}[:name ms/NonBlankString][:creator_id ms/PositiveInt][:skip_if_empty {:optional true} [:maybe :boolean]][:collection_id {:optional true} [:maybe ms/PositiveInt]][:collection_position {:optional true} [:maybe ms/PositiveInt]][:dashboard_id {:optional true} [:maybe ms/PositiveInt]]- [:parameters {:optional true} [:maybe [:sequential :map]]]+ [:parameters {:optional true} [:maybe [:sequential ::pulse.schema/pulse.parameter]]][:disable_links {:optional true} [:maybe ms/BooleanValue]]]](let [pulse-id (create-notification-and-add-cards-and-channels! kvs cards channels)];; return the full Pulse (and record our create event).@@ -592,33 +664,24 @@(retrieve-alert id)))(mu/defn- notification-or-id->existing-card-refs :- [:sequential CardRef]- [notification-or-id]+ [notification-or-id :- NotificationOrId](pulse.db/pulse-card-refs (u/the-id notification-or-id)))(mu/defn- card-refs-have-changed? :- :boolean- [notification-or-id new-card-refs :- [:sequential CardRef]]+ [notification-or-id :- NotificationOrId+ new-card-refs :- [:sequential CardRef]](not= (notification-or-id->existing-card-refs notification-or-id)new-card-refs))-(mu/defn- update-notification-cards-if-changed! [notification-or-id new-card-refs]+(mu/defn- update-notification-cards-if-changed!+ [notification-or-id :- NotificationOrId+ new-card-refs :- [:sequential CardRef]](when (card-refs-have-changed? notification-or-id new-card-refs)(update-notification-cards! notification-or-id new-card-refs)))(mu/defn update-notification!"Update the supplied keys in a `notification`."- [notification :- [:map- [:id ms/PositiveInt]- [:name {:optional true} ms/NonBlankString]- [:alert_condition {:optional true} AlertConditions]- [:alert_above_goal {:optional true} boolean?]- [:alert_first_only {:optional true} boolean?]- [:skip_if_empty {:optional true} boolean?]- [:collection_id {:optional true} [:maybe ms/PositiveInt]]- [:collection_position {:optional true} [:maybe ms/PositiveInt]]- [:cards {:optional true} [:sequential CoercibleToCardRef]]- [:channels {:optional true} [:sequential :map]]- [:archived {:optional true} boolean?]- [:parameters {:optional true} [:maybe [:sequential :map]]]]]+ [notification :- UpdateNotificationInput](pulse.db/update-pulse! (u/the-id notification)(u/select-keys-when notification:present [:collection_id :collection_position :archived]diff --git a/src/metabase/pulse/schema.clj b/src/metabase/pulse/schema.cljindex dbb61e7e0ca..3a2c9b293b4 100644--- a/src/metabase/pulse/schema.clj+++ b/src/metabase/pulse/schema.clj@@ -2,32 +2,22 @@"Malli schemas for the pulse module."(:require[metabase.lib.schema.id :as lib.schema.id]+ [metabase.parameters.schema :as parameters.schema][metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms]))(mr/def ::pulse.parameter- "One entry of the `:parameters` column of a Pulse, decoded."- :map)+ "One entry of the `:parameters` column of a Pulse, decoded. A dashboard subscription stores only the+ filter values it overrides (`:id`/`:value`), not the dashboard's own parameter declaration, so `:type`+ is not always present."+ ::parameters.schema/parameter-with-optional-type)(mr/def ::pulse"A Pulse as selected from the app DB: every column of `:pulse`."- [:map {:closed true}- [:id ::lib.schema.id/pulse]- [:creator_id ::lib.schema.id/user]- [:name [:maybe :string]]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]- [:skip_if_empty :boolean]- [:alert_condition [:maybe :string]]- [:alert_first_only [:maybe :boolean]]- [:alert_above_goal [:maybe :boolean]]- [:collection_id [:maybe ::lib.schema.id/collection]]- [:collection_position [:maybe :int]]- [:archived [:maybe :boolean]]- [:dashboard_id [:maybe ::lib.schema.id/dashboard]]- [:parameters [:sequential ::pulse.parameter]]- [:entity_id :string]- [:disable_links [:maybe :boolean]]])+ [:merge+ ::pulse.update+ [:map {:closed true}+ [:id ::lib.schema.id/pulse]]])(mr/def ::pulse.update"What an update (or insert) of a Pulse accepts: every column of `:pulse` except `id`, all optional."@@ -50,17 +40,10 @@(mr/def ::pulse-card"A PulseCard as selected from the app DB: every column of `:pulse_card`."- [:map {:closed true}- [:id ms/PositiveInt]- [:pulse_id ::lib.schema.id/pulse]- [:card_id ::lib.schema.id/card]- [:position :int]- [:include_csv :boolean]- [:include_xls :boolean]- [:dashboard_card_id [:maybe ::lib.schema.id/card]]- [:entity_id :string]- [:format_rows [:maybe :boolean]]- [:pivot_results [:maybe :boolean]]])+ [:merge+ ::pulse-card.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::pulse-card.update"What an update (or insert) of a PulseCard accepts: every column of `:pulse_card` except `id`, all optional."@@ -76,25 +59,24 @@[:pivot_results {:optional true} [:maybe :boolean]]])(mr/def ::pulse-channel.details- "The `:details` column of a PulseChannel, decoded."- :map)+ "The `:details` column of a PulseChannel, decoded. Shape varies by `:channel_type` (email carries+ `:emails`; slack carries `:channel`/`:channels`/`:channel_id`; both may carry `:include_pdf`/+ `:attachment_only`), so every key is optional here rather than modeled as a `:channel_type`-dispatched+ `:multi`."+ [:map {:closed true}+ [:channel {:optional true} [:maybe :string]]+ [:channels {:optional true} [:maybe :string]]+ [:channel_id {:optional true} [:maybe :string]]+ [:include_pdf {:optional true} [:maybe :boolean]]+ [:attachment_only {:optional true} [:maybe :boolean]]+ [:emails {:optional true} [:maybe [:sequential :string]]]])(mr/def ::pulse-channel"A PulseChannel as selected from the app DB: every column of `:pulse_channel`."- [:map {:closed true}- [:id ms/PositiveInt]- [:pulse_id ::lib.schema.id/pulse]- [:channel_type [:or :keyword :string]]- [:details ::pulse-channel.details]- [:schedule_type [:or :keyword :string]]- [:schedule_hour [:maybe :int]]- [:schedule_day [:maybe :string]]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]- [:schedule_frame [:maybe [:or :keyword :string]]]- [:enabled :boolean]- [:entity_id :string]- [:channel_id [:maybe ms/PositiveInt]]])+ [:merge+ ::pulse-channel.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::pulse-channel.update"What an update (or insert) of a PulseChannel accepts: every column of `:pulse_channel` except `id`, all optional."@@ -114,10 +96,10 @@(mr/def ::pulse-channel-recipient"A PulseChannelRecipient as selected from the app DB: every column of `:pulse_channel_recipient`."- [:map {:closed true}- [:id ms/PositiveInt]- [:pulse_channel_id ms/PositiveInt]- [:user_id ::lib.schema.id/user]])+ [:merge+ ::pulse-channel-recipient.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::pulse-channel-recipient.update"What an update (or insert) of a PulseChannelRecipient accepts: every column of `:pulse_channel_recipient` except `id`, all optional."diff --git a/src/metabase/pulse/send.clj b/src/metabase/pulse/send.cljindex 791a0dc58f9..dcddfb3ac07 100644--- a/src/metabase/pulse/send.clj+++ b/src/metabase/pulse/send.clj@@ -35,8 +35,9 @@(if-not (:id recipient){:type :notification-recipient/raw-value:details {:value (:email recipient)}}- {:type :notification-recipient/user- :user recipient}))+ {:type :notification-recipient/user+ :user_id (:id recipient)+ :user recipient})):http[](do@@ -44,18 +45,15 @@[])))(defn- pc->channel- "Given a pulse channel, return the channel object.-- Only supports HTTP channels for now, returns a map with type key for slack and email"+ "The Channel row of an HTTP pulse channel, or nil for email and Slack pulse channels, which have none."[{channel-type :channel_type :as pulse-channel}]- (if (= :http (keyword channel-type))- (pulse.db/channel (:channel_id pulse-channel))- {:type (keyword "channel" (name channel-type))}))+ (when (= :http (keyword channel-type))+ (pulse.db/channel (:channel_id pulse-channel))))(defn- get-notification-handler[pulse-channel](let [channel (pc->channel pulse-channel)- channel-type (:type channel)]+ channel-type (keyword "channel" (name (:channel_type pulse-channel)))]{:channel_type channel-type:channel channel:recipients (channel-recipients pulse-channel)diff --git a/src/metabase/queries/card.clj b/src/metabase/queries/card.cljindex 547e40a517c..ac485fea9cb 100644--- a/src/metabase/queries/card.clj+++ b/src/metabase/queries/card.clj@@ -40,7 +40,7 @@Note this only reaches parameters whose values come from a Field -- one drawing values from a static list or fromanother card is returned in full, as it is on a dashboard."[card :- ::queries.schema/card- slug->value :- [:maybe [:map-of :any :any]]]+ slug->value :- [:maybe [:map-of :string [:or ms/FieldValue [:sequential ms/FieldValue]]]]](let [;; the same source [[get-param-or-throw]] uses: a native card may carry no `:parameters` at all and be;; described entirely by its template tagsslug->param (into {}@@ -68,7 +68,7 @@"Get param values for the \"old style\" parameters. This mimic's the api/dashboard version except we don't havedashcards to worry about. With `constraints`, values are chain-filtered to the rows those constraints match, thesame way the dashboard version filters on its other parameter values."- [card :- ::queries.schema/card+ [card :- [:maybe ::queries.schema/card]param :- ::parameters.schema/parameterquery-string :- [:maybe :string]constraints :- [:maybe ::chain-filter/constraints]]@@ -84,10 +84,13 @@The source of values could be:- static-list: user defined values list- card: values is result of running a card"- ([card param-key]+ ([card :- ::queries.schema/card+ param-key :- ::lib.schema.parameter/id](card-param-values card param-key nil nil))- ([card param-key query-string]+ ([card :- ::queries.schema/card+ param-key :- ::lib.schema.parameter/id+ query-string :- [:maybe ms/NonBlankString]](card-param-values card param-key query-string nil))([card :- ::queries.schema/card@@ -101,12 +104,14 @@(mu/defn card-param-remapped-value"Fetch the remapped value for the given `value` of parameter with ID `:param-key` of `card`. `constraints` limitsthe rows the remapping is read from."- ([card param-key value]+ ([card :- ::queries.schema/card+ param-key :- ::lib.schema.parameter/id+ value :- [:or ms/FieldValue [:sequential ms/FieldValue]]](card-param-remapped-value card param-key value nil))([card :- ::queries.schema/cardparam-key :- ::lib.schema.parameter/id- value+ value :- [:or ms/FieldValue [:sequential ms/FieldValue]]constraints :- [:maybe ::chain-filter/constraints]](or (let [param (get-param-or-throw card param-key)](custom-values/parameter-remapped-valuediff --git a/src/metabase/queries/card_write_checks.clj b/src/metabase/queries/card_write_checks.cljindex 16c860a361a..f04f249886b 100644--- a/src/metabase/queries/card_write_checks.clj+++ b/src/metabase/queries/card_write_checks.clj@@ -68,30 +68,30 @@"If the query is being modified, check that we have data permissions to run the query."[card-before-updates :- ::queries.schema/cardcard-updates :- ::queries.schema/card]- (when (api/column-will-change? :dataset_query card-before-updates card-updates)+ (when (api/column-will-change? (:dataset_query card-before-updates) (get card-updates :dataset_query ::api/not-provided))(query-perms/check-run-permissions-for-query (dissoc (:dataset_query card-updates) :query-permissions/perms))))(defn- check-allowed-to-change-embedding"You must be a superuser to change the value of `enable_embedding`, `embedding_type` or `embedding_params`. Embedding must beenabled."[card-before-updates card-updates]- (when (or (api/column-will-change? :enable_embedding card-before-updates card-updates)- (api/column-will-change? :embedding_type card-before-updates card-updates)- (api/column-will-change? :embedding_params card-before-updates card-updates))+ (when (or (api/column-will-change? (:enable_embedding card-before-updates) (get card-updates :enable_embedding ::api/not-provided))+ (api/column-will-change? (:embedding_type card-before-updates) (get card-updates :embedding_type ::api/not-provided))+ (api/column-will-change? (:embedding_params card-before-updates) (get card-updates :embedding_params ::api/not-provided)))(embedding.validation/check-embedding-enabled)(api/check-superuser)))(mu/defn- check-allowed-to-move[card-before-update :- ::queries.schema/cardcard-updates :- ::queries.schema/card]- (when (api/column-will-change? :dashboard_id card-before-update card-updates)+ (when (api/column-will-change? (:dashboard_id card-before-update) (get card-updates :dashboard_id ::api/not-provided))(check-allowed-to-remove-from-existing-dashboards card-before-update))(collection/check-allowed-to-change-collection card-before-update card-updates))(mu/defn- check-update-result-metadata-data-perms[card-before-updates :- ::queries.schema/cardcard-updates :- ::queries.schema/card]- (when (api/column-will-change? :result_metadata card-before-updates card-updates)+ (when (api/column-will-change? (:result_metadata card-before-updates) (get card-updates :result_metadata ::api/not-provided))(let [database-id (some :database_id [card-before-updates card-updates])result-metadata (:result_metadata card-updates)](query-perms/check-result-metadata-data-perms database-id result-metadata))))diff --git a/src/metabase/queries/db.clj b/src/metabase/queries/db.cljindex 8c6079ae60b..9852fa51df2 100644--- a/src/metabase/queries/db.clj+++ b/src/metabase/queries/db.clj@@ -5,8 +5,10 @@(:require[metabase.app-db.core :as mdb][metabase.dashboards.schema :as dashboards.schema]+ [metabase.lib-be.schema :as lib-be.schema][metabase.lib.schema.id :as lib.schema.id][metabase.queries.schema :as queries.schema]+ [metabase.query-processor.schema][metabase.util.honey-sql-2 :as h2x][metabase.util.malli :as mu][metabase.util.malli.schema :as ms]@@ -488,7 +490,12 @@(mu/defn insert-queries!"Insert the Query `rows`, returning the number inserted."- [rows :- [:sequential :map]]+ [rows :- [:sequential [:map {:closed true}+ [:query [:or+ ::lib-be.schema/empty-query+ :metabase.query-processor.schema/any-query]]+ [:query_hash bytes?]+ [:average_execution_time number?]]]](t2/insert! :model/Query rows))(mu/defn query-hash-statuses-reduciblediff --git a/src/metabase/queries/metadata.clj b/src/metabase/queries/metadata.cljindex 2c2db3c26ea..a34adebcde9 100644--- a/src/metabase/queries/metadata.clj+++ b/src/metabase/queries/metadata.clj@@ -4,7 +4,6 @@[clojure.set :as set][metabase.api.common :as api][metabase.lib-be.core :as lib-be]- [metabase.lib-be.schema :as lib-be.schema][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.lib.schema :as lib.schema]@@ -30,7 +29,7 @@permissions; there was a specific option where you could give a Perms Group permissions to run existing Cards withnative queries, but not to create new ones. With the advent of what is currently being called 'Space-AgePermissions', all Cards' permissions are based on their parent Collection, removing the need for native read perms."- [db :- [:map]]+ [db :- :metabase.warehouses.schema/database](if (and (not (:is_audit db))(= :query-builder-and-native(perms/full-database-permission-for-user@@ -101,7 +100,7 @@Options:- `include-sensitive-fields?` - if true, includes fields with visibility_type :sensitive (default false)"[queries :- [:maybe [:sequential ::lib.schema/query]]- opts :- [:maybe [:map [:include-sensitive-fields? {:optional true} :boolean]]]]+ opts :- [:maybe [:map {:closed true} [:include-sensitive-fields? {:optional true} :boolean]]]](let [source-table-ids (into #{}(mapcat lib/all-source-table-ids)queries)@@ -224,13 +223,9 @@(mu/defn batch-fetch-dashboard-metadata"Fetch dependent metadata for dashboards."- [dashboards :- [:sequential- [:map {:optional true} [:dashcards- [:sequential- [:map- [:card {:optional true} [:maybe ::queries.schema/card]]- [:series {:optional true} [:maybe [:sequential [:map- [:dataset_query ::lib-be.schema/maybe-legacy-or-empty-query]]]]]]]]]]]+ [dashboards :- [:sequential [:or+ :metabase.dashboards.schema/dashboard+ :metabase.xrays.automagic-dashboards.schema/dashboard]]](let [dashcards (mapcat :dashcards dashboards)cards (for [{:keys [card series]} dashcards:let [all (conj series card)]diff --git a/src/metabase/queries/models/card.clj b/src/metabase/queries/models/card.cljindex 954ceb77746..5de79e6addf 100644--- a/src/metabase/queries/models/card.clj+++ b/src/metabase/queries/models/card.clj@@ -47,6 +47,7 @@[metabase.util.i18n :refer [tru]][metabase.util.log :as log][metabase.util.malli :as mu]+ [metabase.util.malli.fn :as mu.fn][methodical.core :as methodical][toucan2.core :as t2][toucan2.pipeline :as t2.pipeline]@@ -310,7 +311,7 @@Pass false when the query itself is not changing, so that an unrelated update (rename, archive, ...) doesn't wipea previously-valid table_id just because the derivation can no longer resolve it (e.g. the source card wasdeleted)."- ([card]+ ([card :- ::queries.schema/card](populate-query-fields card true))([{query :dataset_query, :as card} :- ::queries.schema/cardclear-stale-table-id? :- :boolean]@@ -388,7 +389,7 @@(defn- invalid-dashboard-internal-card-update-reason?"Returns the reason, if any, why this card is an invalid Dashboard Question"[card changes]- (let [dq-will-change? (api/column-will-change? :dashboard_id card changes)+ (let [dq-will-change? (api/column-will-change? (:dashboard_id card) (get changes :dashboard_id ::api/not-provided))will-be-dq? (or (and (not dq-will-change?)(:dashboard_id card))(and dq-will-change?@@ -396,12 +397,12 @@(when will-be-dq?(cond(not (or dq-will-change?- (not (api/column-will-change? :collection_id card changes))))+ (not (api/column-will-change? (:collection_id card) (get changes :collection_id ::api/not-provided)))))(tru "Invalid Dashboard Question: Cannot manually set `collection_id` on a Dashboard Question")- (api/column-will-change? :collection_position card changes)+ (api/column-will-change? (:collection_position card) (get changes :collection_position ::api/not-provided))(tru "Invalid Dashboard Question: Cannot set `collection_position` on a Dashboard Question");; `column-will-change?` seems broken in the case where we 'change' :question to "question"- (and (api/column-will-change? :type card changes)+ (and (api/column-will-change? (:type card) (get changes :type ::api/not-provided))(not (contains? #{"question" :question} (:type changes))))(tru "Invalid Dashboard Question: Cannot set `type` on a Dashboard Question")))))@@ -675,29 +676,33 @@(assoc card :dimensions dimensions :dimension_mappings dimension-mappings))card))+(defn- plausible-card-select?+ "Whether `card` looks like it was SELECTed as a real Card row (as opposed to some sort of odd query, like an+ aggregation over cards, that happens to run through the same after-select hook)."+ [card]+ (boolean (and (:id card)+ (or (:dataset_query card)+ (:result_metadata card)+ (:database_id card)+ (:type card)))))+(mu/defn- upgrade-card-schema-to-latest :- ::queries.schema/card- [card :- :map]- (-> (if (and (:id card)- (or (:dataset_query card)- (:result_metadata card)- (:database_id card)- (:type card)))- ;; A plausible select to run the after-select logic on.- (if-not (:card_schema card)- ;; Plausible but no :card_schema - error.- (throw (ex-info "Cannot SELECT a Card without including :card_schema"- {:card-id (:id card)}))- ;; Plausible and has the schema, so run the upgrades over it.- (loop [card card]- ;; Use >= to allow for downgrades.- (if (>= (:card_schema card) current-schema-version)- card- (let [new-version (inc (:card_schema card))]- (recur (assoc (upgrade-card-schema-to card new-version)- :card_schema new-version))))))- ;; Some sort of odd query like an aggregation over cards. Just return it as-is.- card)- queries.schema/normalize-card))+ "Run the schema upgrades over a plausible Card row and normalize it. Only call this+ when [[plausible-card-select?]] is true; other queries against `:report_card` should be returned as-is."+ [card :- ::queries.schema/card]+ (-> (if-not (:card_schema card)+ ;; Plausible but no :card_schema - error.+ (throw (ex-info "Cannot SELECT a Card without including :card_schema"+ {:card-id (:id card)}))+ ;; Plausible and has the schema, so run the upgrades over it.+ (loop [card card]+ ;; Use >= to allow for downgrades.+ (if (>= (:card_schema card) current-schema-version)+ card+ (let [new-version (inc (:card_schema card))]+ (recur (assoc (upgrade-card-schema-to card new-version)+ :card_schema new-version))))))+ (->> (lib/normalize ::queries.schema/card))))(defonce ^:private unique-cards-with-blank-dataset-query(atom #{}))@@ -754,16 +759,22 @@public-sharing/remove-public-uuid-if-public-sharing-is-disabledadd-query-description-to-metric-card;; At this point, the card should be at schema version 20 or higher.- upgrade-card-schema-to-latest+ (cond-> (plausible-card-select? card) upgrade-card-schema-to-latest)monitor-blank-dataset-query))+(defn- normalize-card+ "`card` normalized to `::queries.schema/card`, checked against that schema wherever `mu/defn`s are instrumented."+ [card]+ (cond->> (lib/normalize ::queries.schema/card card)+ (mu.fn/instrument-ns? *ns*) (mu.fn/validate-output {:fn-name `normalize-card} [:maybe ::queries.schema/card])))+(t2/define-before-insert :model/Card[card](u/prog1(-> card(assoc :metabase_version config/mb-version-string:card_schema current-schema-version)- queries.schema/normalize-card+ normalize-card;; Must have an entity_id before populating the metadata. TODO (Cam 7/11/25) -- actually, this is no longer true,;; since we're removing `:ident`s; we can probably remove this now.(u/assoc-default :entity_id (u/generate-nano-id))@@ -793,7 +804,9 @@"If we have fresh result_metadata, we don't have to populate it anew. When result_metadata doesn'tchange for a native query, populate-result-metadata removes it (set to nil) unless prevented by theverified-result-metadata? flag (see #37009)."- [card changes verified-result-metadata?]+ [card :- ::queries.schema/card+ changes :- [:maybe ::queries.schema/card]+ verified-result-metadata? :- [:maybe :boolean]](-> (cond-> card(or (empty? (:result_metadata card))(not verified-result-metadata?)@@ -816,8 +829,8 @@(t2/define-before-update :model/Card[{:keys [verified-result-metadata?] :as card}]- (let [changes (some-> card t2/changes queries.schema/normalize-card)- card (queries.schema/normalize-card card)]+ (let [changes (some-> card t2/changes normalize-card)+ card (normalize-card card)](collection/check-allowed-content (:type card) (:collection_id changes))(-> card(dissoc :verified-result-metadata?)@@ -900,13 +913,13 @@dashboard-tab-id :dashboard_tab_idarchived-update :archived}delete-old-dashcards?]- (let [dashboard-changes? (api/column-will-change? :dashboard_id card-before-update card-updates)+ (let [dashboard-changes? (api/column-will-change? old-dashboard-id (get card-updates :dashboard_id ::api/not-provided))new-dashboard-id (if-not dashboard-changes?old-dashboard-iddashboard-id-update)on-dashboard-before? (boolean old-dashboard-id)on-dashboard-after? (boolean new-dashboard-id)- archived-changes? (api/column-will-change? :archived card-before-update card-updates)+ archived-changes? (api/column-will-change? old-archived (get card-updates :archived ::api/not-provided))new-archived (if-not archived-changes?old-archivedarchived-update)@@ -1045,7 +1058,7 @@{:missing-keys (apply disj(set (keys after))(set (keys before)))})))- (boolean (some #(do (api/column-will-change? % before after)) (keys after)))))+ (boolean (some (fn [k] (api/column-will-change? (get before k) (get after k))) (keys after)))))(def ^:private card-compare-keys"When comparing a card to possibly unverify, only consider these keys as changing something 'important' about the@@ -1203,7 +1216,7 @@[{:keys [card-before-update card-updates actor delete-old-dashcards?]}];; don't block our precious core.async thread, run the actual DB updates on a separate thread(t2/with-transaction [_conn]- (api/maybe-reconcile-collection-position! card-before-update card-updates)+ (api/maybe-reconcile-collection-position! (select-keys card-before-update [:collection_id :collection_position]) (select-keys card-updates [:collection_id :collection_position]))(autoplace-or-remove-dashcards-for-card! card-before-update card-updates delete-old-dashcards?)(let [updated-fields (u/select-keys-when card-updates;; `collection_id` and `description` can be `nil` (in order to unset them).@@ -1255,8 +1268,7 @@(mu/defn fully-parameterized?"Given a Card, returns `true` if its query is fully parameterized."- [{query :dataset_query, :as _card} :- [:map- [:dataset_query [:maybe [:or ::lib.schema/query ::lib-be.schema/empty-query]]]]]+ [{query :dataset_query, :as _card} :- ::queries.schema/card](if (empty? query)true(lib/fully-parameterized-query? query)))diff --git a/src/metabase/queries/models/card/metadata.clj b/src/metabase/queries/models/card/metadata.cljindex 69b2973a907..098e5173890 100644--- a/src/metabase/queries/models/card/metadata.clj+++ b/src/metabase/queries/models/card/metadata.clj@@ -5,11 +5,13 @@[metabase.analyze.core :as analyze][metabase.api.common :as api][metabase.lib-be.core :as lib-be]+ [metabase.lib-be.schema :as lib-be.schema][metabase.lib.core :as lib][metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.metadata :as lib.schema.metadata][metabase.models.interface :as mi][metabase.queries.db :as queries.db]+ [metabase.queries.schema :as queries.schema][metabase.query-processor.metadata :as qp.metadata][metabase.query-processor.preprocess :as qp.preprocess][metabase.query-processor.schema :as qp.schema]@@ -24,7 +26,7 @@[:fn {:error/message "A future"} future?])(mu/defn- legacy-result-metadata-future :- ::future- [query :- :map]+ [query :- ::lib-be.schema/maybe-legacy-or-empty-query](future(try;; card result_metadata is persisted in legacy shape; Lib-shape migration pending@@ -46,9 +48,19 @@ saved later when it is ready."[:map[:metadata-future ::future]]])+(def ^:private ModelResultMetadataOptions+ "Options accepted by [[maybe-async-model-result-metadata]]."+ [:map {:closed true}+ [:original-query {:optional true} [:maybe ::lib-be.schema/maybe-legacy-or-empty-query]]+ [:query {:optional true} [:maybe ::lib-be.schema/maybe-legacy-or-empty-query]]+ [:metadata {:optional true} analyze/ResultsMetadata]+ [:original-metadata {:optional true} analyze/ResultsMetadata]+ [:model? {:optional true} [:maybe :boolean]]+ [:entity-id {:optional true} [:maybe :string]]+ [:valid-metadata? {:optional true} [:maybe :boolean]]])+(mu/defn- maybe-async-model-result-metadata :- ::maybe-async-result-metadata- [{:keys [query metadata original-metadata valid-metadata?]} :- [:map- [:valid-metadata? :any]]]+ [{:keys [query metadata original-metadata valid-metadata?]} :- ModelResultMetadataOptions](log/debug "Querying for metadata and blending model metadata")(let [futur (-> querylegacy-result-metadata-future)@@ -72,7 +84,7 @@ saved later when it is ready."{:metadata (combiner result)})))(mu/defn- maybe-async-recomputed-metadata :- ::maybe-async-result-metadata- [query]+ [query :- ::lib-be.schema/maybe-legacy-or-empty-query](log/debug "Querying for metadata")(let [futur (legacy-result-metadata-future query)result (deref futur metadata-sync-wait-ms ::timed-out)]@@ -100,7 +112,14 @@ saved later when it is ready."This is also complicated because everything is optional, so we cannot assume the client will provide metadata andmight need to save a metadata edit, or might need to use db-saved metadata on a modified dataset."- [{:keys [original-query query metadata original-metadata model?], :as options}]+ [{:keys [original-query query metadata original-metadata model?], :as options}+ :- [:map {:closed true}+ [:original-query {:optional true} [:maybe ::lib-be.schema/maybe-legacy-or-empty-query]]+ [:query {:optional true} [:maybe ::lib-be.schema/maybe-legacy-or-empty-query]]+ [:metadata {:optional true} analyze/ResultsMetadata]+ [:original-metadata {:optional true} analyze/ResultsMetadata]+ [:model? {:optional true} [:maybe :boolean]]+ [:entity-id {:optional true} [:maybe :string]]]](let [valid-metadata? (and metadata(mr/validate analyze/ResultsMetadata metadata))](cond@@ -144,9 +163,9 @@ saved later when it is ready.""Save metadata when (and if) it is ready. Takes a chan that will eventually return metadata. Waits upto [[metadata-async-timeout-ms]] for the metadata, and then saves it if the query of the card has not changed."[result-metadata-future :- ::future- card :- [:map+ card :- [:map {:closed true}[:id ::lib.schema.id/card]- [:dataset_query :map]]]+ [:dataset_query ::lib-be.schema/maybe-legacy-or-empty-query]]](let [id (u/the-id card)](future(try@@ -214,10 +233,11 @@ saved later when it is ready."[:result_metadata {:optional true} [:maybe [:sequential ::lib.schema.metadata/lib-or-legacy-column]]]]"When inserting/updating a Card, populate the result metadata column if not already populated by inferring themetadata from the query."- ([card]+ ([card :- ::queries.schema/card](populate-result-metadata card nil))- ([{query :dataset_query metadata :result_metadata :as card} changes]+ ([{query :dataset_query metadata :result_metadata :as card} :- ::queries.schema/card+ changes :- [:maybe ::queries.schema/card]](-> (cond;; not updating the query => no-op(and (not-empty changes)diff --git a/src/metabase/queries/models/parameter_card.clj b/src/metabase/queries/models/parameter_card.cljindex c96806113d2..a150cc92c66 100644--- a/src/metabase/queries/models/parameter_card.clj+++ b/src/metabase/queries/models/parameter_card.clj@@ -82,7 +82,7 @@(mu/defn check-parameter-source-card-permissions"Read-check the Cards `parameters` draw their values from."- [parameters :- [:maybe [:sequential :map]]]+ [parameters :- [:maybe [:sequential ::parameters.schema/parameter]]](doseq [card-id (values-source-card-ids parameters)](api/read-check :model/Card card-id)))diff --git a/src/metabase/queries/schema.clj b/src/metabase/queries/schema.cljindex 6f3ee9e336e..84c7047659e 100644--- a/src/metabase/queries/schema.clj+++ b/src/metabase/queries/schema.clj@@ -1,16 +1,17 @@(ns metabase.queries.schema(:require+ [metabase.content-verification.schema][metabase.documents.schema :as documents.schema][metabase.lib-be.schema :as lib-be.schema][metabase.lib-metric.schema :as lib-metric.schema]- [metabase.lib.core :as lib][metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.metadata :as lib.schema.metadata][metabase.parameters.schema :as parameters.schema]+ [metabase.permissions.schema :as permissions.schema][metabase.util.log :as log]- [metabase.util.malli :as mu][metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms]+ [metabase.warehouse-schema.schema][potemkin :as p]))(p/import-vars@@ -36,26 +37,112 @@nil)))}:any]])+(mr/def ::card.result-metadata.model-override+ "A model's result metadata column as serialization exports it: only the keys a user can override, which the Card's+ hooks merge onto the metadata computed from its query."+ [:map {:closed true}+ [:name :string]+ [:id {:optional true} [:maybe ::lib.schema.id/field]]+ [:description {:optional true} [:maybe :string]]+ [:display_name {:optional true} [:maybe :string]]+ [:semantic_type {:optional true} [:maybe ms/FieldSemanticOrRelationType]]+ [:fk_target_field_id {:optional true} [:maybe ::lib.schema.id/field]]+ [:settings {:optional true} [:maybe ms/VisualizationSettings]]+ [:visibility_type {:optional true} [:maybe [:or :keyword :string]]]+ [:lib/source_display_name {:optional true} [:maybe :string]]])+(mr/def ::card- "Schema for an instance of a `:model/Card` (everything is optional to support updates)."- [:map- [:id {:optional true} [:maybe ::lib.schema.id/card]]- [:collection_id {:optional true} [:maybe ::lib.schema.id/collection]]- [:dashboard_id {:optional true} [:maybe ::lib.schema.id/dashboard]]- [:database_id {:optional true} [:maybe ::lib.schema.id/database]]- [:document_id {:optional true} [:maybe ::documents.schema/document.id]]- [:dataset_query {:optional true} [:maybe ::lib-be.schema/maybe-legacy-or-empty-query]]- [:description {:optional true} [:maybe :string]]- [:name {:optional true} [:maybe :string]]- [:parameters {:optional true} [:maybe [:ref ::parameters.schema/parameters]]]- [:parameter_mappings {:optional true} [:maybe [:ref ::parameters.schema/parameter-mappings]]]- [:type {:optional true} [:maybe ::card.type]]- [:result_metadata {:optional true} [:maybe [:ref ::card.result-metadata]]]])+ "Schema for an instance of a `:model/Card`: every real column of `:report_card` (see `::card.update`) plus `:id`,+ the `:persisted/*` columns some queries join in from `persisted_info`, and the keys some callers hydrate onto a+ Card before passing it here."+ [:merge+ ::card.update+ [:map {:closed true}+ [:id {:optional true} [:maybe ::lib.schema.id/card]]+ [:persisted/active {:optional true} [:maybe :boolean]]+ [:persisted/definition {:optional true} [:maybe :string]]+ [:persisted/query_hash {:optional true} [:maybe :string]]+ [:persisted/state {:optional true} [:maybe :string]]+ [:persisted/table_name {:optional true} [:maybe :string]]+ [:dashboardcard_id {:optional true} [:maybe ::lib.schema.id/dashcard]]+ [:collection {:optional true} [:maybe :metabase.collections.schema/collection-or-root]]+ [:creator {:optional true} [:maybe :metabase.users.schema/user]]+ [:dashboard {:optional true} [:maybe [:ref :metabase.dashboards.schema/dashboard]]]+ [:document {:optional true} [:maybe ::documents.schema/document]]+ [:last-edit-info {:optional true} [:maybe+ [:map {:closed true}+ [:timestamp [:maybe ms/TemporalInstant]]+ [:id [:maybe ms/PositiveInt]]+ [:first_name [:maybe :string]]+ [:last_name [:maybe :string]]+ [:email [:maybe :string]]]]]+ [:moderation_reviews {:optional true} [:sequential :metabase.content-verification.schema/moderation-review]]+ [:can_delete {:optional true} :boolean]+ [:can_manage_db {:optional true} :boolean]+ [:can_restore {:optional true} :boolean]+ [:can_write {:optional true} :boolean]+ [:dashboard_count {:optional true} :int]+ [:query_description {:optional true} [:maybe :string]]+ [:parameter_usage_count {:optional true} :int]+ [:average_query_time {:optional true} [:maybe number?]]+ [:last_query_start {:optional true} [:maybe ms/TemporalInstant]]+ [:based_on_upload {:optional true} [:maybe ::lib.schema.id/table]]+ [:param_fields {:optional true} [:maybe [:map-of :string [:sequential ::param-field]]]]+ [:is_remote_synced {:optional true} :boolean]+ [:authority_level {:optional true} [:maybe [:or :keyword :string]]]+ [:collection_name {:optional true} [:maybe :string]]+ [:collection_authority_level {:optional true} [:maybe [:or :keyword :string]]]+ [:dashboard_name {:optional true} [:maybe :string]]+ [:entity-coll-id {:optional true} [:maybe ::lib.schema.id/collection]]+ [:moderated-status {:optional true} [:maybe :string]]+ [:moderated_status {:optional true} [:maybe :string]]+ [:location {:optional true} [:maybe :string]]+ [:dashboard_tab_id {:optional true} [:maybe ms/PositiveInt]]+ [:in_library {:optional true} :boolean]+ [:persisted {:optional true} :boolean]+ [:query_average_duration {:optional true} [:maybe number?]]+ [:download_perms {:optional true} [:maybe [:enum :none :limited :full]]]+ [:in_dashboards {:optional true} [:maybe [:sequential+ [:map {:closed true}+ [:id ::lib.schema.id/dashboard]+ [:name :string]+ [:collection_id [:maybe ::lib.schema.id/collection]]+ [:description [:maybe :string]]+ [:archived :boolean]+ [:enable_embedding :boolean]]]]]]])++(mr/def ::param-field.name-field+ "A Field trimmed to the columns a parameter widget needs, as the `:name_field` hydration attaches it."+ [:map {:closed true}+ [:id ::lib.schema.id/field]+ [:table_id [:maybe ::lib.schema.id/table]]+ [:display_name [:maybe :string]]+ [:base_type [:maybe [:or :keyword :string]]]+ [:effective_type [:maybe [:or :keyword :string]]]+ [:name :string]+ [:semantic_type [:maybe [:or :keyword :string]]]+ [:has_field_values [:maybe [:or :keyword :string]]]+ [:fk_target_field_id [:maybe ::lib.schema.id/field]]+ [:settings [:maybe :metabase.warehouse-schema.schema/field.settings]]])++(mr/def ::param-field.target+ "A `::param-field.name-field` further hydrated with its own `:name_field`, as the `:target` hydration attaches it."+ [:merge+ ::param-field.name-field+ [:map {:closed true}+ [:name_field {:optional true} [:maybe ::param-field.name-field]]]])-(mu/defn normalize-card :- [:maybe ::card]- "Normalize a `card` so it satisfies the `::card` schema."- [card :- [:maybe :map]]- (lib/normalize ::card card))+(mr/def ::param-field+ "A Field trimmed down to the columns a parameter widget needs, as `:param_fields` hydrates it onto a Card or+ Dashboard."+ [:merge+ ::param-field.target+ [:map {:closed true}+ [:target {:optional true} [:maybe ::param-field.target]]+ [:dimensions {:optional true} [:sequential [:merge+ :metabase.warehouse-schema.schema/dimension+ [:map {:closed true}+ [:human_readable_field {:optional true} [:maybe ::param-field.name-field]]]]]]]])(mr/def ::card.dataset-query"The `:dataset_query` column of a Card, decoded."@@ -85,7 +172,7 @@"What an update (or insert) of a Card accepts: every column of `:report_card` except `id`, all optional, plus `:verified-result-metadata?` consumed by the model's hooks."[:map {:closed true}[:created_at {:optional true} [:maybe ms/TemporalInstant]]- [:updated_at {:optional true} [:maybe ms/TemporalInstant]]+ [:updated_at {:optional true} [:maybe [:or ms/TemporalInstant [:= :updated_at]]]][:name {:optional true} [:maybe :string]][:description {:optional true} [:maybe :string]][:display {:optional true} [:maybe [:or :keyword :string]]]@@ -102,8 +189,10 @@[:enable_embedding {:optional true} [:maybe :boolean]][:embedding_params {:optional true} [:maybe ms/EmbeddingParams]][:cache_ttl {:optional true} [:maybe :int]]- [:result_metadata {:optional true} [:maybe ::card.result-metadata]]- [:collection_position {:optional true} [:maybe :int]]+ [:result_metadata {:optional true} [:maybe [:or+ ::card.result-metadata+ [:sequential ::card.result-metadata.model-override]]]]+ [:collection_position {:optional true} [:maybe [:or :int [:tuple [:enum :+ :-] [:= :collection_position] [:= 1]]]]][:entity_id {:optional true} [:maybe :string]][:parameters {:optional true} [:maybe [:sequential ::card.parameter]]][:parameter_mappings {:optional true} [:maybe [:sequential ::card.parameter-mapping]]]@@ -131,14 +220,10 @@(mr/def ::parameter-card"A ParameterCard as selected from the app DB: every column of `:parameter_card`."- [:map {:closed true}- [:id ms/PositiveInt]- [:updated_at ms/TemporalInstant]- [:created_at ms/TemporalInstant]- [:card_id ::lib.schema.id/card]- [:parameterized_object_type [:or :keyword :string]]- [:parameterized_object_id ms/PositiveInt]- [:parameter_id :string]])+ [:merge+ ::parameter-card.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::parameter-card.update"What an update (or insert) of a ParameterCard accepts: every column of `:parameter_card` except `id`, all optional."@@ -152,14 +237,13 @@(mr/def ::query.query"The `:query` column of a Query, decoded."- :map)+ [:ref ::lib-be.schema/maybe-legacy-or-internal-query])(mr/def ::query"A Query as selected from the app DB: every column of `:query`."- [:map {:closed true}- [:query_hash [:or bytes? :string]]- [:average_execution_time :int]- [:query [:maybe ::query.query]]])+ [:merge+ ::query.update+ [:map {:closed true}]])(mr/def ::query.update"What an update (or insert) of a Query accepts: every column of `:query` except `id`, all optional."@@ -170,48 +254,16 @@(mr/def ::query-execution.lens-params"The `:lens_params` column of a QueryExecution, decoded."- :map)+ [:map {:closed true}+ [:join_step {:optional true} [:maybe :int]]])(mr/def ::query-execution"A QueryExecution as selected from the app DB: every column of `:query_execution`, plus `:row_count` added by the model's after-select hook."- [:map {:closed true}- [:id ms/PositiveInt]- [:hash [:or bytes? :string]]- [:started_at ms/TemporalInstant]- [:running_time :int]- [:result_rows :int]- [:native :boolean]- [:context [:maybe [:or :keyword :string]]]- [:error [:maybe :string]]- [:executor_id [:maybe ::lib.schema.id/user]]- [:card_id [:maybe ::lib.schema.id/card]]- [:dashboard_id [:maybe ::lib.schema.id/dashboard]]- [:pulse_id [:maybe ::lib.schema.id/pulse]]- [:database_id [:maybe ::lib.schema.id/database]]- [:cache_hit [:maybe :boolean]]- [:action_id [:maybe ::lib.schema.id/action]]- [:is_sandboxed [:maybe :boolean]]- [:cache_hash [:maybe [:or bytes? :string]]]- [:embedding_client [:maybe :string]]- [:embedding_sdk_version [:maybe :string]]- [:parameterized [:maybe :boolean]]- [:transform_id [:maybe ::lib.schema.id/transform]]- [:lens_id [:maybe :string]]- [:lens_params [:maybe ::query-execution.lens-params]]- [:auth_method [:maybe [:or :keyword :string]]]- [:tenant_id [:maybe ms/PositiveInt]]- [:is_impersonated [:maybe :boolean]]- [:is_db_routed [:maybe :boolean]]- [:parameters [:maybe :string]]- [:embedding_hostname [:maybe :string]]- [:embedding_path [:maybe :string]]- [:user_agent [:maybe :string]]- [:ip_address [:maybe :string]]- [:sanitized_user_agent [:maybe :string]]- [:embedding_route [:maybe :string]]- [:metabase_version [:maybe :string]]- [:embedding_client_identifier [:maybe :string]]- [:row_count {:optional true} :int]])+ [:merge+ ::query-execution.update+ [:map {:closed true}+ [:id ms/PositiveInt]+ [:row_count {:optional true} :int]]])(mr/def ::query-execution.update"What an update (or insert) of a QueryExecution accepts: every column of `:query_execution` except `id`, all optional."@@ -254,12 +306,10 @@(mr/def ::query-table"A QueryTable as selected from the app DB: every column of `:query_table`."- [:map {:closed true}- [:id ms/PositiveInt]- [:card_id ::lib.schema.id/card]- [:table_id [:maybe ::lib.schema.id/table]]- [:schema [:maybe :string]]- [:table :string]])+ [:merge+ ::query-table.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::query-table.update"What an update (or insert) of a QueryTable accepts: every column of `:query_table` except `id`, all optional."@@ -271,24 +321,18 @@(mr/def ::stored-result.dataset-query"The `:dataset_query` column of a StoredResult, decoded."- :map)+ :metabase.lib.util/query-like)(mr/def ::stored-result.data-access-token"The `:data_access_token` column of a StoredResult, decoded."- :map)+ ::permissions.schema/data-access-token)(mr/def ::stored-result"A StoredResult as selected from the app DB: every column of `:stored_result`."- [:map {:closed true}- [:id ms/PositiveInt]- [:result_data [:or bytes? :string]]- [:creator_id [:maybe ::lib.schema.id/user]]- [:database_id [:maybe ::lib.schema.id/database]]- [:dataset_query ::stored-result.dataset-query]- [:data_access_token [:maybe ::stored-result.data-access-token]]- [:row_count [:maybe :int]]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]])+ [:merge+ ::stored-result.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::stored-result.update"What an update (or insert) of a StoredResult accepts: every column of `:stored_result` except `id`, all optional."@@ -304,13 +348,10 @@(mr/def ::stored-result-use"A StoredResultUse as selected from the app DB: every column of `:stored_result_use`."- [:map {:closed true}- [:id ms/PositiveInt]- [:stored_result_id ms/PositiveInt]- [:exploration_id [:maybe ms/PositiveInt]]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]- [:card_id [:maybe ::lib.schema.id/card]]])+ [:merge+ ::stored-result-use.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::stored-result-use.update"What an update (or insert) of a StoredResultUse accepts: every column of `:stored_result_use` except `id`, all optional."diff --git a/src/metabase/queries_rest/api/card.clj b/src/metabase/queries_rest/api/card.cljindex 6cffc390257..6f0a2ca012b 100644--- a/src/metabase/queries_rest/api/card.clj+++ b/src/metabase/queries_rest/api/card.clj@@ -264,7 +264,7 @@(map #(update-keys % u/->kebab-case-en) cols))(mu/defn- source-cols- [card+ [card :- ::queries.schema/cardsource :- [:enum ::breakouts ::aggregations]](if-let [names (get-in card [:visualization_settings (case source::breakouts :graph.dimensions@@ -342,7 +342,7 @@Provide `page-size` to limit the number of cards returned, it does not guaranteed to return exactly `page-size` cards.Use `fetch-compatible-series` for that."- [card :- :map+ [card :- ::queries.schema/card{:keys [query last-cursor page-size exclude-ids] :as _options}:- [:map {:closed true}[:query {:optional true} [:maybe ms/NonBlankString]]@@ -550,8 +550,8 @@(contains? card-updates :dashboard_id))(queries/actual-collection-id card-updates))](cond-> card-updates- (or (api/column-will-change? :dashboard_id card-before-update card-updates)- (api/column-will-change? :collection_id card-before-update card-updates))+ (or (api/column-will-change? (:dashboard_id card-before-update) (get card-updates :dashboard_id ::api/not-provided))+ (api/column-will-change? (:collection_id card-before-update) (get card-updates :collection_id ::api/not-provided)))(assoc :collection_id collection-id))))(mu/defn update-card!@@ -589,9 +589,8 @@(not (= :list (keyword (get card-updates :display))))){:display :table})(when (and- (api/column-will-change? :dashboard_id- card-before-update- card-updates)+ (api/column-will-change? (:dashboard_id card-before-update)+ (get card-updates :dashboard_id ::api/not-provided))(:dashboard_id card-updates))(api/check-400(not (:archived card-updates)))diff --git a/src/metabase/query_permissions/impl.clj b/src/metabase/query_permissions/impl.cljindex 24496b63f35..3f202e39b65 100644--- a/src/metabase/query_permissions/impl.clj+++ b/src/metabase/query_permissions/impl.clj@@ -28,6 +28,7 @@[metabase.util.i18n :refer [tru]][metabase.util.log :as log][metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms][metabase.util.match :as match][toucan2.core :as t2]))@@ -65,6 +66,10 @@;; {:perms/view-data {table-id :unrestricted}} source-card-read-perms;;+(mr/def ::query+ "A query at any stage of QP processing, carrying the internal keys that QP middleware, drivers, and sandboxing own."+ [:map {:closed false, ::mr/deliberately-open true, :description "a query at any stage of QP processing"}])+(defn- merge-source-ids"Merge function which takes the union of two sets of IDs, if they are both sets"[val1 val2]@@ -81,6 +86,41 @@:else (throw (ex-info "Don't know how to merge values!"{:val1 val1 :val2 val2}))))+(defn- source-ids*+ "Recursive walk backing [[query->source-ids]], over legacy or MBQL 5 queries and the nested-query maps it dissocs its way into."+ [query parent-source-card-id in-sandbox?]+ (if (:lib/type query)+ ;; convert MBQL 5 to legacy+ ;;+ ;; legacy usage -- don't do things like this going forward+ #_{:clj-kondo/ignore [:discouraged-var]}+ (recur (lib/->legacy-MBQL query) parent-source-card-id in-sandbox?)+ ;; already legacy MBQL+ (apply merge-with merge-source-ids+ (match/match-many query+ (:and m {:qp/stage-is-from-source-card (id :guard identity)})+ (merge-with merge-source-ids+ {:card-ids #{id}}+ (source-ids* (dissoc m :qp/stage-is-from-source-card) id in-sandbox?))++ (:and m {:query-permissions/sandboxed-table (id :guard identity)})+ (merge-with merge-source-ids+ {:table-ids #{id}}+ (when-not (or parent-source-card-id in-sandbox?)+ {:table-query-ids #{id}})+ (source-ids* (dissoc m :query-permissions/sandboxed-table :native) parent-source-card-id true))++ {:native &truthy}+ (when-not parent-source-card-id+ {:native? true})++ (:and m {:source-table (id :guard pos-int?)})+ (merge-with merge-source-ids+ {:table-ids #{id}}+ (when-not (or parent-source-card-id in-sandbox?)+ {:table-query-ids #{id}})+ (source-ids* (dissoc m :source-table) parent-source-card-id in-sandbox?))))))+(mu/defn query->source-ids :- [:maybe[:map[:table-ids {:optional true} [:set ::lib.schema.id/table]]@@ -116,48 +156,13 @@Add the table to the table-ids set. If there's no parent-source-card-id, also add itto the table-query-ids set, then continue the match."- ([query]- (query->source-ids query nil false))-- ([query :- :map ; this works on either legacy or MBQL 5 but also on inner queries or other nested maps (it calls itself recursively)- parent-source-card-id :- [:maybe ::lib.schema.id/card]- in-sandbox? :- :any]- (if (:lib/type query)- ;; convert MBQL 5 to legacy- ;;- ;; legacy usage -- don't do things like this going forward- #_{:clj-kondo/ignore [:discouraged-var]}- (recur (lib/->legacy-MBQL query) parent-source-card-id in-sandbox?)- ;; already legacy MBQL- (apply merge-with merge-source-ids- (match/match-many query- (:and m {:qp/stage-is-from-source-card (id :guard identity)})- (merge-with merge-source-ids- {:card-ids #{id}}- (query->source-ids (dissoc m :qp/stage-is-from-source-card) id in-sandbox?))-- (:and m {:query-permissions/sandboxed-table (id :guard identity)})- (merge-with merge-source-ids- {:table-ids #{id}}- (when-not (or parent-source-card-id in-sandbox?)- {:table-query-ids #{id}})- (query->source-ids (dissoc m :query-permissions/sandboxed-table :native) parent-source-card-id true))-- {:native &truthy}- (when-not parent-source-card-id- {:native? true})-- (:and m {:source-table (id :guard pos-int?)})- (merge-with merge-source-ids- {:table-ids #{id}}- (when-not (or parent-source-card-id in-sandbox?)- {:table-query-ids #{id}})- (query->source-ids (dissoc m :source-table) parent-source-card-id in-sandbox?)))))))+ [query :- ::query]+ (source-ids* query nil false))(mu/defn query->source-table-ids"Returns a sequence of all :source-table IDs referenced by a query. Convenience wrapper around `query->source-ids` ifonly table ID information is needed. "- [query :- :map]+ [query :- ::query](when (seq query)(:table-ids (query->source-ids query))))@@ -219,13 +224,13 @@([[preprocess-without-per-user-lens]]), so the result is identical for every user. THROWS when thequery cannot be preprocessed — e.g. a card in the source chain has been deleted — so callersgating cached reads on it can fail closed instead of treating the query as touching nothing."- [query :- :map]+ [query :- ::query](when (seq query)(query->source-ids (preprocess-without-per-user-lens query))))(mu/defn query->resolved-source-table-ids :- [:maybe [:set ::lib.schema.id/table]]"The Table IDs of [[query->resolved-source-ids]]. Throws on an unpreprocessable query, as it does."- [query :- :map]+ [query :- ::query](:table-ids (query->resolved-source-ids query)))(defn- referenced-card-ids@@ -380,10 +385,25 @@(let [result-metadata (:result_metadata (card database-id card-id))](check-result-metadata-data-perms database-id result-metadata)))+(mr/def ::perm-value+ "A required (or granted) permission level: either a single level, or a level per Table (Table 0 is a sentinel used+ by [[legacy-mbql-required-perms]] for \"no such Table\" when perms calculation itself fails)."+ [:or :keyword [:map-of [:or ::lib.schema.id/table [:= 0]] :keyword]])++(mr/def ::required-perms+ "The permissions required to run a query, as returned by [[required-perms-for-query]]."+ [:map {:closed true}+ [:paths {:optional true} [:set :string]]+ [:card-ids {:optional true} [:set ::lib.schema.id/card]]+ [:perms/view-data {:optional true} ::perm-value]+ [:perms/create-queries {:optional true} ::perm-value]])+(mu/defn has-perm-for-query? :- :boolean"Returns true when the query is accessible for the given perm-type and required-perms for individual tables, or theentire DB, false otherwise. Only throws if the permission format is incorrect."- [{db-id :database :as _query} perm-type required-perms]+ [{db-id :database :as _query} :- ::query+ perm-type :- [:enum :perms/view-data :perms/create-queries]+ required-perms :- ::required-perms](boolean(if-let [db-or-table-perms (perm-type required-perms)](cond@@ -436,14 +456,14 @@being answered as one: a failure to work out which permissions `query` needs (otherwise logged aterror and folded into the answer), and any other error the checks raise, such as a missing Card in[[check-card-read-perms]]. A denial still returns `false`."- ([query]+ ([query :- ::query](can-run-query? query false))- ([query :- :map+ ([query :- ::queryalready-preprocessed? :- :boolean](can-run-query? query already-preprocessed? false))- ([{database-id :database :as query} :- :map+ ([{database-id :database :as query} :- ::queryalready-preprocessed? :- :booleanthrow-non-permission-errors? :- :boolean](try@@ -486,7 +506,7 @@(mu/defn check-run-permissions-for-query"Make sure the Current User has the appropriate permissions to run `query`. We don't want Users saving Cards withqueries they wouldn't be allowed to run!"- [query :- :map]+ [query :- ::query]{:pre [(map? query)]}(let [query (dissoc query :query-permissions/perms)expanded (trydiff --git a/src/metabase/query_processor.clj b/src/metabase/query_processor.cljindex 9602986d475..b463e1db15a 100644--- a/src/metabase/query_processor.clj+++ b/src/metabase/query_processor.clj@@ -80,7 +80,7 @@(mu/defn process-query :- [:fn {:error/message "process-query unexpectedly returned nil."} some?]"Process an MBQL query. This is the main entrypoint to the magical realm of the Query Processor."- ([query]+ ([query :- ::qp.schema/any-query](process-query query nil))([query :- ::qp.schema/any-query@@ -102,7 +102,7 @@3. A few extra keys like `:running_time` and `:started_at` are added to the QPresponse (see [[process-userland-query/process-userland-query-middleware]])"- ([query]+ ([query :- ::qp.schema/any-query](userland-query query nil))([query :- ::qp.schema/any-query@@ -120,7 +120,7 @@see [[qp.constraints/add-default-userland-constraints]].This ultimately powers most of the REST API entrypoints into the QP."- ([query]+ ([query :- ::qp.schema/any-query](userland-query-with-default-constraints query nil))([query :- ::qp.schema/any-querydiff --git a/src/metabase/query_processor/api.clj b/src/metabase/query_processor/api.cljindex 7254e10b80d..d1cbd501bf4 100644--- a/src/metabase/query_processor/api.clj+++ b/src/metabase/query_processor/api.clj@@ -61,7 +61,11 @@[query :- [:or ::lib.schema/query ::lib-be.schema/internal-query]& {:keys [context export-format was-pivot]:or {context :ad-hoc- export-format :api}}]+ export-format :api}}+ :- [:maybe [:map {:closed true}+ [:context {:optional true} ::lib.schema.info/context]+ [:export-format {:optional true} ::qp.schema/export-format]+ [:was-pivot {:optional true} [:maybe :boolean]]]]](span/with-span!{:name "run-query-async"};; store table id trivially iff we get a query with simple source-table@@ -114,7 +118,7 @@in `export-format`.(export-format->context :json) ;-> :json-download"- [export-format]+ [export-format :- ::qp.schema/export-format](keyword (str (u/qualified-name export-format) "-download")))(def ^:private column-ref-regex #"^\[.+\]$")diff --git a/src/metabase/query_processor/card.clj b/src/metabase/query_processor/card.cljindex 9979e89d700..90ebdc0a3ff 100644--- a/src/metabase/query_processor/card.clj+++ b/src/metabase/query_processor/card.clj@@ -9,8 +9,10 @@[metabase.events.core :as events][metabase.lib.core :as lib][metabase.lib.schema :as lib.schema]+ [metabase.lib.schema.constraints :as lib.schema.constraints][metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.info :as lib.schema.info]+ [metabase.lib.schema.middleware-options :as lib.schema.middleware-options][metabase.lib.schema.parameter :as lib.schema.parameter][metabase.lib.schema.template-tag :as lib.schema.template-tag][metabase.parameters.schema :as parameters.schema]@@ -68,7 +70,7 @@(mapv (fn [{:keys [target], :as parameter}](cond-> parameter(:stage-number (lib/parameter-target-dimension-options target))- (update :target lib/update-parameter-target-dimension-options assoc :stage-number -1)))+ (update :target lib/update-parameter-target-dimension-options #(assoc % :stage-number -1))))parameters))(mu/defn- last-stage-number@@ -83,7 +85,7 @@(and (= param-type :temporal-unit)(lib/parameter-target-is-dimension? target)(nil? (:stage-number (lib/parameter-target-dimension-options target))))- (update :target lib/update-parameter-target-dimension-options assoc :stage-number -2)))+ (update :target lib/update-parameter-target-dimension-options #(assoc % :stage-number -2))))parameters))(mu/defn query-for-card :- [:maybe ::lib.schema/query]@@ -92,9 +94,11 @@card-type :type:as card} :- ::queries.schema/cardparameters :- [:maybe ::parameters.schema/parameters]- constraints :- [:maybe :map]- middleware :- [:maybe :map]- & [ids]]+ constraints :- [:maybe ::lib.schema.constraints/constraints]+ middleware :- [:maybe ::lib.schema.middleware-options/middleware-options]+ & [ids] :- [:* [:maybe [:map {:closed true}+ [:dashboard-id {:optional true} [:maybe ::lib.schema.id/dashboard]]+ [:dashcard-id {:optional true} [:maybe ::lib.schema.id/dashcard]]]]]](when (seq dataset-query)(let [stage-numbers (explict-stage-references parameters)explicit-stage-numbers? (boolean (seq stage-numbers))@@ -387,13 +391,24 @@`card-transform` is applied after the Card read check and must preserve the Card's identity. Result metadata from atransformed query is returned but not persisted to the Card."[card :- ::queries.schema/card- export-format+ export-format :- ::qp.schema/export-format& {:keys [parameters constraints context dashboard-id dashcard middleware qp make-run ignore-cache card-transform]:or {constraints (qp.constraints/default-query-constraints)context :question;; param `make-run` can be used to control how the query is ran, e.g. if you need to customize the `context`;; passed to the QP- make-run process-query-for-card-default-run-fn}}]+ make-run process-query-for-card-default-run-fn}}+ :- [:maybe [:map {:closed true}+ [:parameters {:optional true} [:maybe ::parameters.schema/parameters-with-optional-types]]+ [:constraints {:optional true} [:maybe ::lib.schema.constraints/constraints]]+ [:context {:optional true} [:maybe ::lib.schema.info/context]]+ [:dashboard-id {:optional true} [:maybe ::lib.schema.id/dashboard]]+ [:dashcard {:optional true} [:maybe [:ref :metabase.dashboards.schema/dashboard-card]]]+ [:middleware {:optional true} [:maybe ::lib.schema.middleware-options/middleware-options]]+ [:qp {:optional true} [:maybe ifn?]]+ [:make-run {:optional true} [:maybe ifn?]]+ [:ignore-cache {:optional true} [:maybe :boolean]]+ [:card-transform {:optional true} [:maybe ifn?]]]]]{:pre [(map? card) (pos-int? (:id card)) (u/maybe? sequential? parameters)]}(let [card (api/read-check card)stored-query (:dataset_query card)@@ -403,7 +418,7 @@(assoc :skip-result-metadata-persistence? true))card-id (:id card)dashcard-id (:id dashcard)- parameters (some-> parameters parameters.schema/normalize-parameters-without-adding-default-types)+ parameters (some->> parameters (lib/normalize ::parameters.schema/parameters-with-optional-types))parameters (enrich-parameters-from-card parameters (combined-parameters-and-template-tags card))dash-viz (when (and (not= context :question) dashcard)(:visualization_settings dashcard))@@ -415,6 +430,7 @@(or qp process-query-for-card-default-qp))runner (make-run qp export-format)query (-> (query-for-card card parameters constraints middleware {:dashboard-id dashboard-id})+ api/check-404(assoc :viz-settings merged-viz)(update :middleware (fn [middleware](mergediff --git a/src/metabase/query_processor/compile.clj b/src/metabase/query_processor/compile.cljindex 82de4f63bb8..cc9b1e25f68 100644--- a/src/metabase/query_processor/compile.clj+++ b/src/metabase/query_processor/compile.clj@@ -18,15 +18,17 @@(mr/def ::compiled"Compiled query and parameters (SQL or whatever native query language)."- [:map- [:query :any]- [:params {:optional true} [:maybe [:sequential :any]]]])+ [:map {::mr/deliberately-open true+ :description "drivers add their own keys to a compiled query, e.g. Mongo's `:collection`, `:projections` and `:mbql?` or BigQuery's `:qp/table-name`, and an already-native query compiles to its whole native stage"}+ [:query ::lib.schema/driver-native-query]+ [:params {:optional true} [:maybe [:sequential :metabase.lib.schema.common/field-value]]]])(mr/def ::compiled-with-inlined-parameters"Query with inlined parameters (:params must be empty)"- [:map- [:query :any]- [:params {:optional true} [:maybe [:sequential {:max 0} :any]]]])+ [:map {::mr/deliberately-open true+ :description "drivers add their own keys to a compiled query, e.g. Mongo's `:collection`, `:projections` and `:mbql?` or BigQuery's `:qp/table-name`"}+ [:query ::lib.schema/driver-native-query]+ [:params {:optional true} [:maybe [:sequential {:max 0} :metabase.lib.schema.common/field-value]]]])(mr/def ::query-with-compiled-query"An MBQL 5 query that also has a compiled native query attached (unless it was already a native-only query in the firstdiff --git a/src/metabase/query_processor/dashboard.clj b/src/metabase/query_processor/dashboard.cljindex e73d87b8721..bd594eed1b1 100644--- a/src/metabase/query_processor/dashboard.clj+++ b/src/metabase/query_processor/dashboard.clj@@ -7,8 +7,10 @@[metabase.api.common :as api][metabase.dashboards.schema :as dashboards.schema][metabase.events.core :as events]+ [metabase.lib-be.core :as lib-be][metabase.lib-metric.core :as lib-metric][metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata][metabase.lib.schema.id :as lib.schema.id][metabase.parameters.params :as params][metabase.permissions.core :as perms]@@ -79,7 +81,9 @@(lib/mbql-stage? query -1))card(if-let [dimension (default-metric-dimension provider query card)]- (let [definition (lib-metric/from-metric-metadata provider card)+ (let [definition (lib-metric/from-metric-metadata+ provider+ (lib.metadata/metric (lib-be/application-database-metadata-provider (:database_id card)) (:id card)))breakout (lib-metric/dimension-breakout definition dimension)](cond-> cardbreakout (assoc :dataset_query (-> query@@ -166,9 +170,9 @@that those parameters exist and have allowed types, and merge in default values and other info from the parametermappings."[dashboard :- ::dashboards.schema/dashboard- dashcard :- ::dashboards.schema/dashcard+ dashcard :- ::dashboards.schema/dashboard-cardcard-id :- ::lib.schema.id/card- request-params :- [:maybe [:sequential :map]]]+ request-params :- [:maybe ::dashboards.schema/parameters]](let [dashboard-id (:id dashboard)dashcard-id (:id dashcard)_ (log/tracef "Resolving Dashboard %d Card %d query request parameters" dashboard-id card-id)@@ -235,12 +239,13 @@(perms/most-permissive-database-permission-for-userapi/*current-user-id* :perms/view-data(:database_id card))))- (let [resolved-params (resolve-params-for-query dashboard dashcard card-id parameters)+ (let [resolved-params (resolve-params-for-query dashboard dashcard card-id+ (some->> parameters not-empty (lib/normalize ::dashboards.schema/parameters)))options (merge{:ignore-cache false:constraints (qp.constraints/default-query-constraints):context :dashboard}- (dissoc options :dashboard :card)+ (dissoc options :dashboard :card :export-format){:parameters resolved-params:dashboard-id dashboard-id})](log/tracef "Running Query for Dashboard %d, Card %d, Dashcard %d"diff --git a/src/metabase/query_processor/db.clj b/src/metabase/query_processor/db.cljindex 05f2e61aa80..cf4d6ba412f 100644--- a/src/metabase/query_processor/db.clj+++ b/src/metabase/query_processor/db.clj@@ -5,6 +5,7 @@[java-time.api :as t][metabase.app-db.core :as app-db][metabase.lib.schema.id :as lib.schema.id]+ [metabase.lib.schema.metadata :as lib.schema.metadata][metabase.queries.schema :as queries.schema][metabase.util.malli :as mu][metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]@@ -17,7 +18,7 @@(mu/defn table"The Table with `table-id`, or nil."- [table-id :- ::lib.schema.id/table]+ [table-id :- [:maybe ::lib.schema.id/table]](t2/select-one :model/Table :id table-id {:from [(warehouse-schema-overlay/table-query)]}))(mu/defn source-card-metadata@@ -68,7 +69,7 @@(mu/defn set-card-result-metadata!"Set the result metadata of the Card with `card-id` without touching `updated_at`."[card-id :- ::lib.schema.id/card- result-metadata :- [:maybe [:sequential :map]]]+ result-metadata :- [:maybe ::lib.schema.metadata/card.result-metadata]](t2/update! :model/Card card-id {:result_metadata result-metadata:updated_at :updated_at}))diff --git a/src/metabase/query_processor/metadata.clj b/src/metabase/query_processor/metadata.cljindex 9562bc87f32..c287eaaac10 100644--- a/src/metabase/query_processor/metadata.clj+++ b/src/metabase/query_processor/metadata.clj@@ -3,7 +3,7 @@MBQL queries; for native queries we use the driver implementation of[[metabase.driver/query-result-metadata]], which hopefully can calculate metadata without running the query. Ifthat's not possible, our fallback `:default` implementation adds the equivalent of `LIMIT 1` to query and runs it."- (:refer-clojure :exclude [mapv not-empty])+ (:refer-clojure :exclude [mapv not-empty select-keys])(:require[metabase.analyze.core :as analyze][metabase.driver :as driver]@@ -19,16 +19,16 @@[metabase.util :as u][metabase.util.log :as log][metabase.util.malli :as mu]- [metabase.util.performance :as perf :refer [mapv not-empty]]))+ [metabase.util.performance :as perf :refer [mapv not-empty select-keys]]))(mu/defn- metadata-from-preprocessing :- [:maybe [:sequential :map]]"For MBQL queries or native queries with result metadata attached to them already we can infer the columns just bypreprocessing the query/looking at the last stage of the query."- [query :- :map]+ [query :- ::qp.schema/any-query](not-empty (u/ignore-exceptions (qp.preprocess/query->expected-cols query))))(mu/defn- query-with-limit-1 :- :map- [query :- :map]+ [query :- ::qp.schema/any-query];; for purposes of calculating the actual Fields & types returned by this query we really only need the first;; row in the results(let [query (-> query@@ -42,7 +42,7 @@(assoc-in query [:info :query-hash] (qp.util/query-hash query))))(mu/defn- result-metadata-rff :- ::qp.schema/rff- [metadata]+ [metadata :- ::qp.schema/metadata](let [cols (:cols metadata)](fn rf([]@@ -56,7 +56,7 @@(mu/defmethod driver/query-result-metadata :default :- [:sequential :map][_driver :- :keyword- query :- :map]+ query :- ::qp.schema/any-query](let [query' (query-with-limit-1 query)](try(qp/process-query query' result-metadata-rff)@@ -68,7 +68,7 @@"Get metadata from the driver's implementation of [[metabase.driver/query-result-metadata]]. For JDBC-based driversthis returns metadata without actually running queries; the default implementation will run the query with `LIMIT 1`to get results."- [query :- :map+ [query :- ::qp.schema/any-querycurrent-user-id :- [:maybe ::lib.schema.id/user]](let [query (cond-> querycurrent-user-id (assoc-in [:info :executed-by] current-user-id))@@ -76,7 +76,7 @@(driver/query-result-metadata driver query)))(mu/defn- add-extra-column-metadata :- :map- [col :- :map+ [col :- [:or ::lib.schema.metadata/column :metabase.legacy-mbql.schema/legacy-column-metadata]legacy-or-lib :- [:enum ::legacy ::lib]](let [display-name-key (case legacy-or-lib::legacy :display_name@@ -95,7 +95,7 @@(let [legacy-col (cond-> col(= legacy-or-lib ::lib)(perf/update-keys u/->snake_case_en))- semantic-type (analyze/infer-semantic-type-by-name legacy-col)]+ semantic-type (analyze/infer-semantic-type-by-name (select-keys legacy-col [:name :base_type :semantic_type]))](merge col(when semantic-type{semantic-type-key semantic-type}))))]@@ -104,7 +104,8 @@infer-semantic-type-by-name))))(mu/defn- result-metadata* :- [:sequential :map]- [query current-user-id]+ [query :- ::qp.schema/any-query+ current-user-id :- [:maybe ::lib.schema.id/user]](or (metadata-from-preprocessing query)(metadata-from-driver query current-user-id)))@@ -118,10 +119,10 @@Returns columns as Lib-style `kebab-case` column metadata; for legacy metadata you can use [[legacy-result-metadata]]instead."- ([query]+ ([query :- ::qp.schema/any-query](result-metadata query nil))- ([query :- [:map [:database ::lib.schema.id/database]]+ ([query :- ::qp.schema/any-querycurrent-user-id :- [:maybe ::lib.schema.id/user]](mapv(fn [col]@@ -132,7 +133,7 @@(mu/defn- ensure-legacy :- ::qp.schema/result-metadata.column{:deprecated "0.57.0"}- [col :- :map]+ [col :- ::lib.schema.metadata/lib-or-legacy-column](letfn [(ensure-field-ref [col];; HACK for backward compatibility with FE stuff -- ideally we would be able to remove this entirely but;; if we do some e2e tests fail that I don't really have the energy to spend all day debugging -- Cam@@ -152,7 +153,7 @@Note: it is preferable to use [[metabase.lib.core/lib-metadata-column->legacy-metadata-column]] directly if you reallyneed to do this sort of conversion."{:deprecated "0.51.0"}- [query :- :map+ [query :- ::qp.schema/any-querycurrent-user-id :- [:maybe ::lib.schema.id/user]](mapv;; deprecated fn delegating to its deprecated private helper; both leave togetherdiff --git a/src/metabase/query_processor/middleware/add_implicit_joins.clj b/src/metabase/query_processor/middleware/add_implicit_joins.cljindex 1e881f24121..1ec666faf00 100644--- a/src/metabase/query_processor/middleware/add_implicit_joins.clj+++ b/src/metabase/query_processor/middleware/add_implicit_joins.clj@@ -16,6 +16,7 @@[metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.join :as lib.schema.join][metabase.lib.schema.metadata :as lib.schema.metadata]+ [metabase.lib.schema.ref :as lib.schema.ref][metabase.lib.walk :as lib.walk][metabase.query-processor.error-type :as qp.error-type][metabase.util.i18n :refer [tru]]@@ -107,7 +108,7 @@problematic case is when refs with and without `:source-field-name` are mixed, but there should be the same implicitjoin for all of them."[metadata-providerable :- ::lib.schema.metadata/metadata-providerable- {:keys [source-field source-field-name source-field-join-alias]} :- :map] ; not `::lib.schema.ref/field.options` because this might come from a legacy ref+ {:keys [source-field source-field-name source-field-join-alias]} :- ::lib.schema.ref/field.options](let [fk-field (lib.metadata/field metadata-providerable source-field)](m/assoc-some {:fk-field-id source-field}:fk-field-name (when (and (some? source-field-name) (not= source-field-name (:name fk-field)))diff --git a/src/metabase/query_processor/middleware/add_remaps.clj b/src/metabase/query_processor/middleware/add_remaps.cljindex bc010259fe4..e0a8dc37322 100644--- a/src/metabase/query_processor/middleware/add_remaps.clj+++ b/src/metabase/query_processor/middleware/add_remaps.clj@@ -35,6 +35,7 @@[metabase.lib.schema.common :as lib.schema.common][metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.join :as lib.schema.join]+ [metabase.lib.schema.literal :as lib.schema.literal][metabase.lib.schema.metadata :as lib.schema.metadata][metabase.lib.schema.order-by :as lib.schema.order-by][metabase.lib.schema.ref :as lib.schema.ref]@@ -53,7 +54,22 @@[:tuple[:enum :field :expression :aggregation][:and- :map+ [:map {:closed true}+ [:semantic-type {:optional true} [:maybe ::lib.schema.common/semantic-or-relation-type]]+ [:database-type {:optional true} [:maybe ::lib.schema.common/non-blank-string]]+ [:name {:optional true} [:maybe ::lib.schema.common/non-blank-string]]+ [:display-name {:optional true} [:maybe ::lib.schema.common/non-blank-string]]+ [:case-sensitive {:optional true} :boolean]+ [:include-current {:optional true} :boolean]+ [:default {:optional true} [:ref :metabase.lib.schema.expression/expression]]+ [:join-alias {:optional true} [:ref ::lib.schema.join/alias]]+ [:source-field {:optional true} ::lib.schema.id/field]+ [:source-field-name {:optional true} ::lib.schema.common/non-blank-string]+ [:source-field-join-alias {:optional true} ::lib.schema.common/non-blank-string]+ [:temporal-unit {:optional true} [:ref :metabase.lib.schema.temporal-bucketing/unit]]+ [:inherited-temporal-unit {:optional true} [:ref :metabase.lib.schema.temporal-bucketing/unit]]+ [:original-temporal-unit {:optional true} [:ref :metabase.lib.schema.temporal-bucketing/unit]]+ [:binning {:optional true} [:ref :metabase.lib.schema.binning/binning]]][:fn{:error/message "options map without namespaced keys and base-type/effective-type"}(complement (some-fn :base-type :effective-type :lib/uuid))]]@@ -72,7 +88,7 @@(mr/def ::external-remapping"Schema for the info we fetch about `external` type Dimensions that will be used for remappings in this Query. Fetchedby the pre-processing portion of the middleware, and passed along to the post-processing portion."- [:map+ [:map {:closed true}[:id ::lib.schema.id/dimension] ; unique ID for the remapping[:name ::lib.schema.common/non-blank-string] ; display name for the remapping[:field-id ::lib.schema.id/field] ; ID of the Field being remapped@@ -104,15 +120,18 @@(mr/def ::remap-info[:and- [:map+ [:map {:closed true}[:original-field-clause :mbql.clause/field][:new-field-clause [:and:mbql.clause/field[:tuple[:= :field]- [:map- [::new-field-dimension-id ::lib.schema.id/dimension]]- :any]]]+ [:map {:closed true}+ [::new-field-dimension-id ::lib.schema.id/dimension]+ [:lib/uuid {:optional true} ::lib.schema.common/uuid]+ [:source-field {:optional true} ::lib.schema.id/field]+ [:join-alias {:optional true} [:ref ::lib.schema.join/alias]]]+ ::lib.schema.id/field]]][:dimension ::external-remapping]][:fn{:error/message "the new field clause should have the same join alias as the original field clause"}@@ -373,7 +392,7 @@;;;; Post-processing(mr/def ::internal-remapping-info- [:map+ [:map {:closed true};; index of original column[:col-index :int];; names@@ -381,26 +400,26 @@;; I'm not convinced this works if there's already a column with the same name in the results.[:to ::lib.schema.common/non-blank-string];; map of original value -> human readable value- [:value->readable :map]+ [:value->readable [:map-of ::lib.schema.literal/literal ::lib.schema.literal/literal]];; Info about the new column we will tack on to end of `:cols`- [:new-column :map]])+ [:new-column ::qp.schema/result-metadata.column]])(mr/def ::internal-columns-info- [:map+ [:map {:closed true}[:internal-only-dims [:maybe [:sequential ::internal-remapping-info]]];; this is just (map :new-column internal-only-dims)- [:internal-only-cols [:maybe [:sequential :map]]]])+ [:internal-only-cols [:maybe ::qp.schema/result-metadata.columns]]]);;;; Metadata(mu/defn- merge-metadata-for-internally-remapped-column :- [:maybe [:sequential :map]]"If one of the internal remapped columns says it's remapped from this column, merge in the `:remapped_to` info."- [columns :- [:maybe [:sequential :map]]+ [columns :- [:maybe ::qp.schema/result-metadata.columns]{:keys [col-index to]} :- ::internal-remapping-info](update (vec columns) col-index assoc :remapped_to to))(mu/defn- merge-metadata-for-internal-remaps :- [:maybe [:sequential :map]]- [columns :- [:maybe [:sequential :map]]+ [columns :- [:maybe ::qp.schema/result-metadata.columns]{:keys [internal-only-dims]} :- [:maybe ::internal-columns-info]](reducemerge-metadata-for-internally-remapped-column@@ -431,9 +450,9 @@;; :name "NAME";; :display_name "Sender ID"}(mu/defn- merge-metadata-for-externally-remapped-column* :- :map- [columns+ [columns :- [:maybe ::qp.schema/result-metadata.columns]{{::keys [original-field-dimension-id new-field-dimension-id]} :options- :as column} :- :map+ :as column} :- ::qp.schema/result-metadata.column{dimension-id :idfrom-name :field-namefrom-display-name :name@@ -477,13 +496,13 @@(log/trace "Added remapping metadata to column"))))(mu/defn- merge-metadata-for-externally-remapped-column :- [:maybe [:sequential :map]]- [columns :- [:maybe [:sequential :map]] dimension :- ::external-remapping]+ [columns :- [:maybe ::qp.schema/result-metadata.columns] dimension :- ::external-remapping](log/tracef "Merging metadata for external dimension %s" (:id dimension))(mapv #(merge-metadata-for-externally-remapped-column* columns % dimension)columns))(mu/defn- merge-metadata-for-external-remaps :- [:maybe [:sequential :map]]- [columns :- [:maybe [:sequential :map]] remapping-dimensions :- [:maybe [:sequential ::external-remapping]]]+ [columns :- [:maybe ::qp.schema/result-metadata.columns] remapping-dimensions :- [:maybe [:sequential ::external-remapping]]](reducemerge-metadata-for-externally-remapped-columncolumns@@ -494,7 +513,7 @@To get this critical information, this uses the `remapping-dimensions` info saved by the pre-processing portion ofthis middleware for external remappings, and the internal-only remapped columns handled by post-processingmiddleware below for internal columns."- [columns :- [:maybe [:sequential :map]]+ [columns :- [:maybe ::qp.schema/result-metadata.columns]remapping-dimensions :- [:maybe [:sequential ::external-remapping]]internal-cols-info :- [:maybe ::internal-columns-info]](-> columns@@ -601,8 +620,7 @@(mu/defn- add-remapped-to-and-from-metadata"Add remapping info `:remapped_from` and `:remapped_to` to each existing column in the results metadata, and addentries for each newly added column to the end of `:cols`."- [metadata :- [:map- [:cols [:maybe [:sequential :map]]]]+ [metadata :- ::qp.schema/metadataremapping-dimensions :- [:maybe [:sequential ::external-remapping]]{:keys [internal-only-cols], :as internal-cols-info} :- [:maybe ::internal-columns-info]](update metadata :cols (fn [cols]@@ -615,7 +633,8 @@added and each row flowing through needs to include the remapped data for the new column. For external remappingsthe column information needs to be updated with what it's being remapped from and the user specified name for theremapped column."- [{:keys [internal-only-dims]} :- ::internal-columns-info rf]+ [{:keys [internal-only-dims]} :- ::internal-columns-info+ rf :- fn?](if-let [remap-fn (make-row-map-fn internal-only-dims)]((map remap-fn) rf)rf))@@ -623,7 +642,7 @@(mu/defn remap-results :- ::qp.schema/rff"Post-processing middleware. Handles `::external-remaps` added by [[add-remapped-columns-middleware]]; transformsresults and adds additional metadata based on these remaps, as well as internal (human-readable values) remaps."- [{::keys [external-remaps], {:keys [disable-remaps?]} :middleware, :as _query} :- :map+ [{::keys [external-remaps], {:keys [disable-remaps?]} :middleware, :as _query} :- ::lib.schema/queryrff :- ::qp.schema/rff](if disable-remaps?rffdiff --git a/src/metabase/query_processor/middleware/annotate.clj b/src/metabase/query_processor/middleware/annotate.cljindex c5625ce2ecf..e82f2dff1f7 100644--- a/src/metabase/query_processor/middleware/annotate.clj+++ b/src/metabase/query_processor/middleware/annotate.clj@@ -10,7 +10,6 @@[metabase.lib.core :as lib][metabase.lib.metadata.result-metadata :as lib.metadata.result-metadata][metabase.lib.schema :as lib.schema]- [metabase.lib.schema.metadata :as lib.schema.metadata][metabase.query-processor.debug :as qp.debug][metabase.query-processor.middleware.annotate.legacy-helper-fns :as annotate.legacy-helper-fns][metabase.query-processor.reducible :as qp.reducible]@@ -21,9 +20,7 @@[potemkin :as p]))(mr/def ::col- [:map- [:source {:optional true} ::lib.schema.metadata/column.legacy-source]- [:field_ref {:optional true} ::mbql.s/Reference]])+ [:or ::mbql.s/legacy-column-metadata ::mbql.s/driver-column])(mr/def ::qp-results-cased-col"Map where all simple keywords are snake_case, but lib keywords can stay in kebab-case."@@ -40,8 +37,7 @@[:maybe [:sequential ::col]])(mr/def ::metadata- [:map- [:cols {:optional true} ::cols]])+ ::qp.schema/metadata)(mu/defn expected-cols :- [:sequential ::qp-results-cased-col]"Return metadata for columns returned by a MBQL 5 `query`.@@ -50,7 +46,7 @@name and base type). If provided these are merged with the columns the query is expected to return.Note this `initial-cols` is more or less required for native queries unless they have metadata attached."- ([query]+ ([query :- ::lib.schema/query](expected-cols query []))([query :- ::lib.schema/query@@ -74,7 +70,7 @@"Native queries don't have the type information from the original `Field` objects used in the query. If the driverreturned a base type more specific than :type/*, use that; otherwise look at the sample of rows and infer the basetype based on the classes of the values"- [{:keys [cols]} :- :map]+ [{:keys [cols]} :- ::metadata](apply analyze/col-wise(for [{driver-base-type :base_type} cols](if (contains? #{nil :type/*} driver-base-type)@@ -165,5 +161,6 @@metadata returned by the driver's impl of `execute-reducible-query` and (b) column metadata inferred by logic inthis namespace."{:deprecated "0.64.0"}- [legacy-query {initial-cols :cols, :as _initial-metadata} :- [:maybe :map]]+ [legacy-query :- :metabase.lib.util/legacy-query+ {initial-cols :cols, :as _initial-metadata} :- [:maybe ::metadata]](expected-cols (annotate.legacy-helper-fns/legacy-query->mbql5-query legacy-query) initial-cols))diff --git a/src/metabase/query_processor/middleware/annotate/legacy_helper_fns.clj b/src/metabase/query_processor/middleware/annotate/legacy_helper_fns.cljindex ef82686f02d..707e5f02d60 100644--- a/src/metabase/query_processor/middleware/annotate/legacy_helper_fns.clj+++ b/src/metabase/query_processor/middleware/annotate/legacy_helper_fns.clj@@ -23,11 +23,7 @@(mu/defn legacy-inner-query->mbql5-query :- ::lib.schema/query"Convert a legacy `inner-query` to an MBQL 5 query. Requires bound QP store."{:deprecated "0.57.0"}- [inner-query :- [:and- :map- [:fn- {:error/message "Should be a legacy MBQL inner query"}- (some-fn :query :source-table :source-query)]]]+ [inner-query :- :metabase.lib.util/query-like];; existing usage -- don't use going forward#_{:clj-kondo/ignore [:deprecated-var]}(qp.store/cached [:mbql5-query (hash inner-query)]@@ -43,7 +39,7 @@(mu/defn legacy-query->mbql5-query :- ::lib.schema/query"Convert a legacy outer `legacy-query` to an MBQL 5 query. Requires bound QP store."- [legacy-query]+ [legacy-query :- :metabase.lib.util/legacy-query](lib/query(qp.store/metadata-provider);; if this query has a `:native` query added to it already then remove that so we don't get schema validation@@ -60,8 +56,8 @@DEPRECATED: use [[mbql-5-aggregation-name]] going forward."{:deprecated "0.64.0"}- [legacy-inner-query :- :map- legacy-ag-clause]+ [legacy-inner-query :- :metabase.lib.util/query-like+ legacy-ag-clause :- ::mbql.s/Aggregation](let [ag-clause (lib/->mbql5 legacy-ag-clause)](or (::add/desired-alias (lib/options ag-clause))(:name (lib/options ag-clause))diff --git a/src/metabase/query_processor/middleware/auto_bucket_datetimes.clj b/src/metabase/query_processor/middleware/auto_bucket_datetimes.cljindex 93d4a87c874..258cd536947 100644--- a/src/metabase/query_processor/middleware/auto_bucket_datetimes.clj+++ b/src/metabase/query_processor/middleware/auto_bucket_datetimes.clj@@ -10,6 +10,7 @@[metabase.lib.schema :as lib.schema][metabase.lib.schema.common :as lib.schema.common][metabase.lib.schema.id :as lib.schema.id]+ [metabase.lib.schema.metadata :as lib.schema.metadata][metabase.lib.walk :as lib.walk][metabase.util.log :as log][metabase.util.malli :as mu]@@ -18,7 +19,7 @@[metabase.util.performance :refer [select-keys every? some not-empty get-in]]))(mr/def ::column-type-info- [:map+ [:map {:closed true}[:base-type [:maybe ::lib.schema.common/base-type]][:effective-type [:maybe ::lib.schema.common/base-type]][:semantic-type {:optional true} [:maybe ::lib.schema.common/semantic-or-relation-type]]])@@ -36,7 +37,8 @@(mu/defn- unbucketed-fields->field-id->type-info :- [:maybe ::column-id-or-name->type-info]"Fetch a map of Field ID -> type information for the Fields referred to by the `unbucketed-fields`. Return an empty mapfor empty `unbucketed-fields`."- [metadata-providerable unbucketed-fields :- [:maybe [:sequential :mbql.clause/field]]]+ [metadata-providerable :- ::lib.schema.metadata/metadata-providerable+ unbucketed-fields :- [:maybe [:sequential :mbql.clause/field]]](merge;; build map of field-literal-name -> {:base-type base-type}(into {} (for [[_tag opts id-or-name] unbucketed-fields@@ -69,7 +71,7 @@(mu/defn- filter-clause?[query :- ::lib.schema/querystage-path :- ::lib.walk/stage-path- x]+ x :- ::lib.schema.common/clause-tag-candidate](and (lib/clause? x)(when-let [expr-type (try(lib.walk/apply-f-for-stage-at-path lib/type-of query stage-path x)@@ -81,7 +83,7 @@(mu/defn- simple-filter-clause?[query :- ::lib.schema/querystage-path :- ::lib.walk/stage-path- x]+ x :- ::lib.schema.common/clause-tag-candidate](and (filter-clause? query stage-path x)(not (lib/clause-of-type? x #{:and :or :not}))))@@ -97,7 +99,7 @@"Is `x` a clause (or a clause that contains a clause) that we should definitely not autobucket?"[query :- ::lib.schema/querystage-path :- ::lib.walk/stage-path- x]+ x :- ::lib.schema.common/clause-tag-candidate](cond;; do not autobucket clauses in a non-compound filter clause that either:(simple-filter-clause? query stage-path x)diff --git a/src/metabase/query_processor/middleware/cache.clj b/src/metabase/query_processor/middleware/cache.cljindex 3ead65a9e3a..37b6eb0d16a 100644--- a/src/metabase/query_processor/middleware/cache.clj+++ b/src/metabase/query_processor/middleware/cache.clj@@ -16,6 +16,7 @@[metabase.cache.core :as cache][metabase.config.core :as config][metabase.lib.core :as lib]+ [metabase.lib.schema :as lib.schema][metabase.query-processor.middleware.cache-backend.db :as backend.db][metabase.query-processor.middleware.cache-backend.interface :as i][metabase.query-processor.middleware.cache.impl :as impl]@@ -272,9 +273,9 @@- `[::miss nil]` -- no entry; or it's expired, or nearly so, and *this* process won the lease; or it's toostale to serve to anyone. The caller must recompute.- `[::canceled nil]` -- the request was canceled."- [ignore-cache?- query-hash :- bytes?- strategy :- :map+ [ignore-cache? :- [:maybe :boolean]+ query-hash :- bytes?+ strategy :- ::lib.schema/cache-strategyrff :- ::qp.schema/rff](if ignore-cache?[::miss nil]@@ -352,7 +353,8 @@(save-results-xform start-time-ns metadata query-hash cache-strategy (rff metadata)))))))(mu/defn- run-query-with-cache :- :any- [qp {:keys [cache-strategy middleware], :as query} :- ::qp.schema/any-query+ [qp :- fn?+ {:keys [cache-strategy middleware], :as query} :- ::qp.schema/any-queryrff :- ::qp.schema/rff];; Query will already have `info.hash` if it's a userland query. It's not the same hash, because this is calculated;; after normalization, instead of before. This is necessary to make caching work properly with sandboxed users, seediff --git a/src/metabase/query_processor/middleware/catch_exceptions.clj b/src/metabase/query_processor/middleware/catch_exceptions.cljindex 86b212a3d50..35f7dfc9e7c 100644--- a/src/metabase/query_processor/middleware/catch_exceptions.clj+++ b/src/metabase/query_processor/middleware/catch_exceptions.clj@@ -6,6 +6,7 @@[metabase.analytics-interface.core :as analytics][metabase.driver :as driver][metabase.lib.schema.common :as lib.schema.common]+ [metabase.queries.schema :as queries.schema][metabase.query-processor.compile :as qp.compile][metabase.query-processor.error-type :as qp.error-type][metabase.query-processor.middleware.permissions :as qp.perms]@@ -16,6 +17,7 @@[metabase.util.i18n :refer [trs]][metabase.util.log :as log][metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr][metabase.util.performance :refer [some get-in]])(:import(clojure.lang ExceptionInfo)@@ -58,6 +60,18 @@(assoc ((get-method format-exception Throwable) e):state (.getSQLState e)))+(mr/def ::format-exception-result+ "The map shape produced by one of this namespace's private `format-exception` methods."+ [:map {:closed true}+ [:status [:enum :failed :interrupted]]+ [:class {:optional true} (lib.schema.common/instance-of-class Class)]+ [:error {:optional true} [:maybe :string]]+ [:stacktrace {:optional true} [:maybe [:sequential :string]]]+ [:error_type {:optional true} :keyword]+ [:error_is_curated {:optional true} :boolean]+ [:ex-data {:optional true} :metabase.lib.schema.common/exception-data]+ [:state {:optional true} [:maybe :string]]])+;; TODO -- some of this logic duplicates the functionality of `clojure.core/Throwable->map`, we should consider;; whether we can use that more extensively and remove some of this logic(defn- exception-chain@@ -68,7 +82,7 @@(mu/defn- best-top-level-error"In cases where the top-level Exception doesn't have the best error message, return a better one to use instead. Weusually want to show SQLExceptions at the top level since they contain more useful information."- [maps :- [:sequential {:min 1} :map]]+ [maps :- [:sequential {:min 1} ::format-exception-result]](some (fn [m](when (isa? (:class m) SQLException);; Some JDBC drivers (e.g. Databricks) return a stacktrace in the@@ -102,15 +116,28 @@:native (when (qp.perms/current-user-has-adhoc-native-query-perms? query)native)})))+(mr/def ::query-execution-info+ "The in-flight QueryExecution info that userland query processing attaches to exceptions: the columns about to be saved, plus the query and start time."+ [:merge+ ::queries.schema/query-execution.update+ [:map {:closed true}+ [:json_query {:optional true} ::qp.schema/any-query]+ [:start_time_millis {:optional true} :int]]])+(mu/defn- query-execution-info :- :map- [query-execution :- :map]+ [query-execution :- ::query-execution-info](dissoc query-execution :result_rows :hash :executor_id :dashboard_id :pulse_id :native :start_time_millis))+(def ^:private ExtraInfo+ [:map {:closed true}+ [:native {:optional true} [:maybe :metabase.query-processor.compile/compiled]]+ [:preprocessed {:optional true} [:maybe :metabase.lib.schema/query]]])+(mu/defn- format-exception* :- [:map [:status :keyword]]"Format a `Throwable` into the usual userland error-response format."- [query :- :map+ [query :- ::qp.schema/any-query^Throwable e :- (lib.schema.common/instance-of-class Throwable)- extra-info :- [:maybe :map]]+ extra-info :- [:maybe ExtraInfo]](try;; [[metabase.query-processor.middleware.process-userland-query/process-userland-query-middleware]] wraps exceptions;; to add query execution info, unwrap them and format the wrapped onediff --git a/src/metabase/query_processor/middleware/expand_macros.clj b/src/metabase/query_processor/middleware/expand_macros.cljindex 5c8ee0ec80a..2dd2a0ff5a2 100644--- a/src/metabase/query_processor/middleware/expand_macros.clj+++ b/src/metabase/query_processor/middleware/expand_macros.clj@@ -2,7 +2,7 @@"Middleware for expanding LEGACY `:segment` 'macros' in *unexpanded* MBQL queries.(`:segment` forms are expanded into filter clauses.)"- (:refer-clojure :exclude [mapv not-empty get-in])+ (:refer-clojure :exclude [mapv not-empty])(:require[metabase.lib.core :as lib][metabase.lib.filter :as lib.filter]@@ -18,16 +18,11 @@[metabase.util.malli :as mu][metabase.util.malli.registry :as mr][metabase.util.match :as match]- [metabase.util.performance :refer [mapv not-empty get-in]]))+ [metabase.util.performance :refer [mapv not-empty]]));;; "legacy macro" as used below means legacy Segment.(mr/def ::legacy-macro- [:and- [:map- [:lib/type [:enum :metadata/segment]]]- [:multi- {:dispatch :lib/type}- [:metadata/segment ::lib.schema.metadata/segment]]])+ ::lib.schema.metadata/segment)(mr/def ::macro-type[:enum :segment])@@ -51,11 +46,10 @@;;; a legacy Segment has one or more filter clauses.-(mu/defn- segment-definition->stage :- ::lib.schema/stage.mbql+(mu/defn- segment-definition->stage :- [:maybe ::lib.schema/stage.mbql]"Extract the MBQL 5 stage from a segment definition. Segment definitions are always MBQL 5 queries at this point(converted by the segment model's after-select hook), so we just extract the first stage."- [_metadata-providerable :- ::lib.schema.metadata/metadata-providerable- {:keys [definition], :as _legacy-macro} :- ::legacy-macro]+ [{:keys [definition], :as _legacy-macro} :- ::legacy-macro](log/trace "Extracting MBQL 5 stage from segment definition")(first (:stages definition)))@@ -63,7 +57,7 @@"Get the filter(s) associated with a Segment."[legacy-macro :- ::legacy-macro](mapv lib/fresh-uuids- (get-in legacy-macro [:definition :filters])))+ (:filters (segment-definition->stage legacy-macro))))(mr/def ::id->legacy-macro[:map-of pos-int? ::legacy-macro])@@ -75,8 +69,7 @@(let [metadata-type (case macro-type ;; left in case we see a :metric here:segment :metadata/segment)](u/prog1 (into {}- (map (juxt :id (fn [legacy-macro]- (assoc legacy-macro :definition (segment-definition->stage metadata-providerable legacy-macro)))))+ (map (juxt :id identity))(lib.metadata/bulk-metadata-or-throw metadata-providerable metadata-type legacy-macro-ids));; make sure all the IDs exist.(doseq [id legacy-macro-ids]@@ -135,7 +128,7 @@([query :- ::lib.schema/query](expand-macros query 0))- ([query recursion-depth]+ ([query :- ::lib.schema/query recursion-depth :- :int](when (> recursion-depth max-recursion-depth)(throw (ex-info (tru "Segment expansion failed. Check mutually recursive segment definitions."){:type qp.error-type/invalid-query, :query query})))diff --git a/src/metabase/query_processor/middleware/fetch_source_query.clj b/src/metabase/query_processor/middleware/fetch_source_query.cljindex 175f47d7548..eaab95a6358 100644--- a/src/metabase/query_processor/middleware/fetch_source_query.clj+++ b/src/metabase/query_processor/middleware/fetch_source_query.clj@@ -1,5 +1,5 @@(ns metabase.query-processor.middleware.fetch-source-query- (:refer-clojure :exclude [some get-in])+ (:refer-clojure :exclude [some get-in empty?])(:require[metabase.driver :as driver][metabase.driver.ddl.interface :as ddl.i]@@ -20,7 +20,7 @@[metabase.util.i18n :refer [trs tru]][metabase.util.log :as log][metabase.util.malli :as mu]- [metabase.util.performance :refer [some get-in]]+ [metabase.util.performance :refer [some get-in empty?]][weavejester.dependency :as dep]));;; TODO -- consider whether [[normalize-card-query]] should be moved into [[metabase.lib.card]], seems like it would@@ -66,26 +66,28 @@(fn [_query _path-type _path stage-or-join](apply dissoc stage-or-join qp-owned-stage-keys))))(update-stages [stages]- (let [stages (fix-mongodb-first-stage stages)- stages (for [stage stages]- ;; This is for detecting circular refs below, and is later used as part of- ;; permissions enforcement- (assoc stage :qp/stage-is-from-source-card card-id))- ;; TODO (Cam 2026-02-25) Check if attaching the metadata is even necessary anymore- card-metadata (into []- (remove :remapped-from)- (lib.card/card-returned-columns metadata-providerable card))- last-stage (cond-> (last stages)- (seq card-metadata) (assoc :lib/stage-metadata {:lib/type :metadata/results, :columns card-metadata})- ;; This will be applied, if still appropriate, by- ;; the [[metabase.query-processor.middleware.persistence]] middleware- ;;- ;; TODO -- not 100% sure I did this right, there are almost no tests for this- persisted? (assoc :persisted-info/native- (qp.persisted/persisted-info-native-query- (:database-id card)- persisted-info)))]- (conj (vec (butlast stages)) last-stage)))+ (if (empty? stages)+ stages+ (let [stages (fix-mongodb-first-stage stages)+ stages (for [stage stages]+ ;; This is for detecting circular refs below, and is later used as part of+ ;; permissions enforcement+ (assoc stage :qp/stage-is-from-source-card card-id))+ ;; TODO (Cam 2026-02-25) Check if attaching the metadata is even necessary anymore+ card-metadata (into []+ (remove :remapped-from)+ (lib.card/card-returned-columns metadata-providerable card))+ last-stage (cond-> (last stages)+ (seq card-metadata) (assoc :lib/stage-metadata {:lib/type :metadata/results, :columns card-metadata})+ ;; This will be applied, if still appropriate, by+ ;; the [[metabase.query-processor.middleware.persistence]] middleware+ ;;+ ;; TODO -- not 100% sure I did this right, there are almost no tests for this+ persisted? (assoc :persisted-info/native+ (qp.persisted/persisted-info-native-query+ (:database-id card)+ persisted-info)))]+ (conj (vec (butlast stages)) last-stage))))(update-query [query](-> (lib/query metadata-providerable query);; Now that cards' queries can come out of the AppDB already in MBQL 5, complete with `:lib/uuid`s,diff --git a/src/metabase/query_processor/middleware/measures.clj b/src/metabase/query_processor/middleware/measures.cljindex 0527edb8eb9..1ea711e6b20 100644--- a/src/metabase/query_processor/middleware/measures.clj+++ b/src/metabase/query_processor/middleware/measures.clj@@ -14,6 +14,7 @@[metabase.lib.metadata :as lib.metadata][metabase.lib.options :as lib.options][metabase.lib.schema :as lib.schema]+ [metabase.lib.schema.metadata :as lib.schema.metadata][metabase.lib.walk :as lib.walk][metabase.query-processor.error-type :as qp.error-type][metabase.util :as u]@@ -53,7 +54,7 @@:measure-id id:measure-name name}))))-(mu/defn- fetch-measures :- [:map-of pos-int? :map]+(mu/defn- fetch-measures :- [:map-of pos-int? ::lib.schema.metadata/measure]"Fetch measure metadata for the given IDs."[query :- ::lib.schema/querymeasure-ids :- [:set {:min 1} pos-int?]]@@ -77,7 +78,7 @@(mu/defn- expand-measures-in-stage :- ::lib.schema/stage"Replace :measure clauses in a stage with their actual aggregation expressions."[stage :- ::lib.schema/stage- id->measure :- [:map-of pos-int? :map]]+ id->measure :- [:map-of pos-int? ::lib.schema.metadata/measure]](match/replace stage[:measure opts (id :guard pos-int?)](b/conddiff --git a/src/metabase/query_processor/middleware/normalize_query.clj b/src/metabase/query_processor/middleware/normalize_query.cljindex 80eeb269adb..eef76aa556d 100644--- a/src/metabase/query_processor/middleware/normalize_query.clj+++ b/src/metabase/query_processor/middleware/normalize_query.clj@@ -4,7 +4,6 @@[metabase.lib.core :as lib][metabase.lib.metadata.protocols :as lib.metadata.protocols][metabase.lib.schema :as lib.schema]- [metabase.lib.schema.id :as lib.schema.id][metabase.query-processor.error-type :as qp.error-type];; the legacy QP pipeline still conveys the metadata provider via the ambient store; no MBQL 5 path yet^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store]@@ -21,7 +20,7 @@(mu/defn normalize-preprocessing-middleware :- ::lib.schema/query"Preprocessing middleware. Normalize a query, meaning do things like convert keys and MBQL clause tags to kebab-casekeywords. Convert query to MBQL 5 if needed."- [query :- [:map [:database ::lib.schema.id/database]]]+ [query :- :metabase.lib.util/query-like](try(normalize* query)(catch Throwable ediff --git a/src/metabase/query_processor/middleware/optimize_temporal_filters.clj b/src/metabase/query_processor/middleware/optimize_temporal_filters.cljindex 851e290a546..cec7a8e050e 100644--- a/src/metabase/query_processor/middleware/optimize_temporal_filters.clj+++ b/src/metabase/query_processor/middleware/optimize_temporal_filters.clj@@ -212,7 +212,9 @@(lib/relative-datetime (inc (if (= n :current) 0 n)) target-unit))))(mu/defn- date-field-with-day-bucketing?- [query path expr :- [:maybe ::lib.schema.expression/expression]]+ [query :- ::lib.schema/query+ path :- ::lib.walk/stage-path+ expr :- [:maybe ::lib.schema.expression/expression]](and (isa? (lib.walk/apply-f-for-stage-at-path lib/type-of query path expr) :type/Date)(= (lib/raw-temporal-bucket expr) :day)))@@ -244,7 +246,11 @@(lib/negate-boolean-expression optimized))))(mu/defn- optimize-comparison-clause :- [:maybe ::lib.schema.mbql-clause/clause]- [query path optimize-temporal-value-fn [tag opts field temporal-value] new-clause-type :- [:enum :< :>=]]+ [query :- ::lib.schema/query+ path :- ::lib.walk/stage-path+ optimize-temporal-value-fn :- ifn?+ [tag opts field temporal-value] :- [:or :mbql.clause/< :mbql.clause/<= :mbql.clause/> :mbql.clause/>=]+ new-clause-type :- [:enum :< :>=]](b/cond(date-field-with-day-bucketing? query path field)[tag opts (change-temporal-unit-to-default field) (change-temporal-unit-to-default temporal-value)]diff --git a/src/metabase/query_processor/middleware/parameters/mbql.clj b/src/metabase/query_processor/middleware/parameters/mbql.cljindex 7d3b3a4be40..fda517aadd3 100644--- a/src/metabase/query_processor/middleware/parameters/mbql.clj+++ b/src/metabase/query_processor/middleware/parameters/mbql.clj@@ -53,7 +53,7 @@[query :- ::lib.schema/querystage-path :- ::lib.walk/pathparam-type :- ::lib.schema.parameter/type- param-value+ param-value :- ::lib.schema.parameter/parameter.value.scalara-ref :- [:or :mbql.clause/field :mbql.clause/expression]](cond;; LEGACY: :id and :category are widget-types misused as parameter types.diff --git a/src/metabase/query_processor/middleware/process_userland_query.clj b/src/metabase/query_processor/middleware/process_userland_query.cljindex 1466c49b62e..db5a27c5eab 100644--- a/src/metabase/query_processor/middleware/process_userland_query.clj+++ b/src/metabase/query_processor/middleware/process_userland_query.clj@@ -83,7 +83,12 @@;; processed the request (and its dynamic bindings and clock) is long gone. `include-sdk-info` also runs in;; the `before-insert` hook as a safety net for any code path that inserts QueryExecution directly (where;; dynamic vars would still be bound).- execution-info' (add-running-time (analytics.core/include-sdk-info execution-info))]+ json-query (:json_query execution-info)+ execution-info' (-> execution-info+ (dissoc :json_query)+ analytics.core/include-sdk-info+ add-running-time+ (cond-> json-query (assoc :json_query json-query)))](if qp.util/*execute-async?*(grouper/submit! @save-execution-metadata-queue execution-info')(save-execution-metadata!* [execution-info']))))diff --git a/src/metabase/query_processor/middleware/reconcile_breakout_and_order_by_bucketing.clj b/src/metabase/query_processor/middleware/reconcile_breakout_and_order_by_bucketing.cljindex efd24b835d9..8008ca0dfca 100644--- a/src/metabase/query_processor/middleware/reconcile_breakout_and_order_by_bucketing.clj+++ b/src/metabase/query_processor/middleware/reconcile_breakout_and_order_by_bucketing.clj@@ -77,7 +77,9 @@vec))(mu/defn- reconcile-bucketing-in-stage :- [:maybe ::lib.schema/stage]- [query stage-path {breakouts :breakout, order-bys :order-by, :as stage}]+ [query :- ::lib.schema/query+ stage-path :- ::lib.walk/stage-path+ {breakouts :breakout, order-bys :order-by, :as stage} :- ::lib.schema/stage](when (and (seq breakouts)(seq order-bys))(when-let [bucketed-breakouts (not-empty (bucketed-breakouts query stage-path stage))]diff --git a/src/metabase/query_processor/middleware/resolve_joins.clj b/src/metabase/query_processor/middleware/resolve_joins.cljindex af513a97a1f..2a55cfe725f 100644--- a/src/metabase/query_processor/middleware/resolve_joins.clj+++ b/src/metabase/query_processor/middleware/resolve_joins.clj@@ -14,7 +14,7 @@[metabase.util.performance :refer [every? mapv empty?]]))(mu/defn- merge-defaults :- ::lib.schema.join/join- [join]+ [join :- ::lib.schema.join/join](merge {:strategy lib.schema.join/default-strategy}(when (str/starts-with? (:alias join) lib/legacy-default-join-alias){:qp/keep-default-join-alias true})diff --git a/src/metabase/query_processor/middleware/resolve_referenced.clj b/src/metabase/query_processor/middleware/resolve_referenced.cljindex 59ec64a815d..ceb89cebd8c 100644--- a/src/metabase/query_processor/middleware/resolve_referenced.clj+++ b/src/metabase/query_processor/middleware/resolve_referenced.clj@@ -15,7 +15,8 @@[metabase.util.malli :as mu][weavejester.dependency :as dep])(:import- (clojure.lang ExceptionInfo)))+ (clojure.lang ExceptionInfo)+ (weavejester.dependency MapDependencyGraph)))(mu/defn- resolve-referenced-card-resources*"Done for side effects; warm the MetadataProvider."@@ -46,7 +47,7 @@(mu/defn- subquery-graph[metadata-providerable :- ::lib.schema.metadata/metadata-providerable- graph :- :map+ graph :- (lib.schema.common/instance-of-class MapDependencyGraph)init-query :- ::lib.schema/query](letfn [(card-subquery-graph [mp graph card-id](let [card-query (fetch-card-query mp card-id)diff --git a/src/metabase/query_processor/middleware/results_metadata.clj b/src/metabase/query_processor/middleware/results_metadata.cljindex c79bb0efc7b..9c4d5358fc8 100644--- a/src/metabase/query_processor/middleware/results_metadata.clj+++ b/src/metabase/query_processor/middleware/results_metadata.clj@@ -12,6 +12,7 @@[metabase.lib.metadata :as lib.metadata][metabase.lib.schema :as lib.schema][metabase.lib.schema.metadata :as lib.schema.metadata]+ [metabase.queries.schema :as queries.schema][metabase.query-processor.db :as query-processor.db][metabase.query-processor.reducible :as qp.reducible][metabase.query-processor.schema :as qp.schema]@@ -123,7 +124,7 @@insights-col-metadata))(mu/defn- insights-xform :- fn?- [orig-metadata :- [:maybe :map]+ [orig-metadata :- [:maybe ::qp.schema/metadata]record! :- ifn?rf :- ifn?](qp.reducible/combine-additional-reducing-fns@@ -157,9 +158,7 @@(mu/defn store-previous-result-metadata!"Store the previous value of a card's result metadata in the qp.store"- [card :- [:maybe- [:map- [:result_metadata {:optional true} [:maybe [:sequential ::lib.schema.metadata/lib-or-legacy-column]]]]]]+ [card :- [:maybe ::queries.schema/card]](when-let [result-metadata (:result_metadata card)](if-let [error (me/humanize (mr/explain [:sequential ::lib.schema.metadata/lib-or-legacy-column] result-metadata))](log/errorf "Invalid card result metadata, ignoring it: %s" (pr-str error))diff --git a/src/metabase/query_processor/middleware/update_used_cards.clj b/src/metabase/query_processor/middleware/update_used_cards.cljindex 9ac8d2c90b4..66f321c5006 100644--- a/src/metabase/query_processor/middleware/update_used_cards.clj+++ b/src/metabase/query_processor/middleware/update_used_cards.clj@@ -19,7 +19,7 @@(mu/defn- update-used-cards!*[card-id-timestamps :- [:sequential- [:map+ [:map {:closed true}[:id ::lib.schema.id/card][:timestamp (lib.schema.common/instance-of-class java.time.OffsetDateTime)]]]](let [card-id->timestamp (update-vals (group-by :id card-id-timestamps)diff --git a/src/metabase/query_processor/middleware/wrap_value_literals.clj b/src/metabase/query_processor/middleware/wrap_value_literals.cljindex 3f1353f528e..dc6c2665f8c 100644--- a/src/metabase/query_processor/middleware/wrap_value_literals.clj+++ b/src/metabase/query_processor/middleware/wrap_value_literals.clj@@ -8,6 +8,7 @@[metabase.lib.metadata :as lib.metadata][metabase.lib.schema :as lib.schema][metabase.lib.schema.expression :as lib.schema.expression]+ [metabase.lib.schema.mbql-clause :as lib.schema.mbql-clause][metabase.lib.types.isa :as lib.types.isa][metabase.lib.walk :as lib.walk][metabase.query-processor.error-type :as qp.error-type]@@ -386,7 +387,7 @@:semantic_type nil,:database_type \"VARCHAR\",:name \"description\"}]]]"- [mbql :- [:cat :keyword [:* :any]]]+ [mbql :- ::lib.schema.mbql-clause/clause](-> mbqllib/->mbql5(as-> $mbql (binding [*type-info* (fn [_query _path clause]diff --git a/src/metabase/query_processor/parameters/dates.clj b/src/metabase/query_processor/parameters/dates.cljindex bbf9f66b7cd..e482d1f2d28 100644--- a/src/metabase/query_processor/parameters/dates.clj+++ b/src/metabase/query_processor/parameters/dates.clj@@ -165,7 +165,8 @@:unit – finds a matching date unit and merges date unit operations to the result:int-value, :int-value-1 – converts the group value to integer:date, :date1, date2 – converts the group value to absolute date"- [regex :- [:fn {:error/message "regular expression"} m/regexp?] group-labels]+ [regex :- [:fn {:error/message "regular expression"} m/regexp?]+ group-labels :- [:sequential :keyword]](fn [param-value](when-let [regex-result (re-matches regex param-value)](into {} (mapcat expand-parser-groups group-labels (rest regex-result))))))@@ -273,10 +274,14 @@(defn- ->iso-8601-date-time [t](t/format :iso-local-date-time t))+(mr/def ::temporal-unit+ (into [:enum] u.date/add-units))+(mu/defn- range->filter :- :mbql.clause/between- [{:keys [start end]} :- [:map- [:start :any]- [:end :any]]+ [{:keys [start end]} :- [:map {:closed true}+ [:start (lib.schema.common/instance-of-class Temporal)]+ [:end (lib.schema.common/instance-of-class Temporal)]+ [:unit {:optional true} ::temporal-unit]]field-clause :- :mbql.clause/field](lib/between (with-temporal-unit-if-field field-clause :day) (->iso-8601-date start) (->iso-8601-date end)))@@ -400,27 +405,33 @@"Returns the first successfully decoded value, run through both parser and a range/filter decoder depending on`decoder-type`. This generates an *inclusive* range by default. The range is adjusted to be exclusive as needed: seedox for [[date-string->range]] for more details."- [decoders- decoder-type :- [:enum :range :filter]- decoder-param- date-string :- :string]+ [decoders :- [:sequential [:map {:closed true}+ [:parser fn?]+ [:range {:optional true} fn?]+ [:filter {:optional true} fn?]]]+ decoder-type :- [:enum :range :filter]+ decoder-param :- [:or [:maybe (lib.schema.common/instance-of-class java.time.LocalDateTime)] :mbql.clause/field :mbql.clause/expression]+ date-string :- :string](some (fn [{parser :parser, parser-result-decoder decoder-type}](when-let [parser-result (and parser-result-decoder (parser date-string))](parser-result-decoder parser-result decoder-param)))decoders))-(mr/def ::temporal-unit- (into [:enum] u.date/add-units))-(mr/def ::temporal-range- [:map+ [:map {:closed true}[:start {:optional true} (lib.schema.common/instance-of-class Temporal)][:end {:optional true} (lib.schema.common/instance-of-class Temporal)][:unit ::temporal-unit]])+(mr/def ::inclusive-options+ [:map {:closed true}+ [:inclusive-start? {:optional true} [:maybe :boolean]]+ [:inclusive-end? {:optional true} [:maybe :boolean]]])+(mu/defn- adjust-inclusive-range-if-needed :- [:maybe ::temporal-range]"Make an inclusive date range exclusive as needed."- [temporal-range :- [:maybe ::temporal-range] {:keys [inclusive-start? inclusive-end?]}]+ [temporal-range :- [:maybe ::temporal-range]+ {:keys [inclusive-start? inclusive-end?]} :- ::inclusive-options](-> temporal-range(m/update-existing :start #(if inclusive-start?%@@ -488,12 +499,12 @@Note that some ranges are open-ended on one side, and will have only a `:start` or an `:end`.";; 1-arg version returns inclusive start/end; 2-arg version can adjust as needed- ([date-string]+ ([date-string :- ::lib.schema.common/non-blank-string](date-string->range date-string nil))([date-string :- ::lib.schema.common/non-blank-string{:keys [inclusive-start? inclusive-end?]- :or {inclusive-start? true inclusive-end? true}}]+ :or {inclusive-start? true inclusive-end? true}} :- [:maybe ::inclusive-options]](let [options {:inclusive-start? inclusive-start?, :inclusive-end? inclusive-end?}](-> (date-string->raw-range date-string)(adjust-inclusive-range-if-needed options)@@ -562,7 +573,8 @@This function is meant to be used for generating inclusive intervals for `:type/DateTime` field filters.* End-exclusive gte lt filters are generated for `:type/DateTime` fields."- [raw-date-str field-type]+ [raw-date-str :- ::lib.schema.common/non-blank-string+ field-type :- ::lib.schema.common/base-type](let [;; `raw-date-str` is sanitized in case it contains millis and timezone which are incompatible;; with [[date-string->range]]. `substitute-field-filter-test` expects that to happen.[range-raw unit] (try (let [r (date-string->raw-range raw-date-str)]diff --git a/src/metabase/query_processor/parameters/operators.clj b/src/metabase/query_processor/parameters/operators.cljindex b69d2a8b0e6..ded1fbd94c2 100644--- a/src/metabase/query_processor/parameters/operators.clj+++ b/src/metabase/query_processor/parameters/operators.clj@@ -12,17 +12,20 @@for [[qp.error-type]], which maybe belongs in Lib too!"(:refer-clojure :exclude [get-in])(:require+ [malli.core :as mc]+ [malli.util :as mut][metabase.lib.core :as lib][metabase.lib.schema.expression :as lib.schema.expression][metabase.lib.schema.parameter :as lib.schema.parameter][metabase.query-processor.error-type :as qp.error-type][metabase.util.i18n :refer [tru]][metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr][metabase.util.match :as match][metabase.util.performance :refer [get-in]]))(mu/defn- operator-arity :- [:maybe [:enum :unary :binary :variadic]]- [param-type]+ [param-type :- ::lib.schema.parameter/type](get-in lib.schema.parameter/types [param-type :operator]))(defn operator?@@ -32,8 +35,8 @@(mu/defn- verify-type-and-arity[field :- [:or :mbql.clause/field :mbql.clause/expression]- param-type- param-value]+ param-type :- ::lib.schema.parameter/type+ param-value :- [:sequential :metabase.lib.schema.common/field-value]](letfn [(maybe-arity-error [n](when (not= n (count param-value))(throw (ex-info (format "Operations Invalid arity: expected %s but received %s"@@ -75,10 +78,16 @@(nil? l) (assoc :type :number/<=, :value [u])))param))+(mr/def ::field-filter-operator-param+ "A parameter built during field filter substitution, whose `:target` already wraps an MBQL 5 ref."+ (mut/assoc (first (mc/children (mr/resolve-schema ::lib.schema.parameter/parameter)))+ :target+ [:tuple [:= :dimension] [:or :mbql.clause/field :mbql.clause/expression]]))+(mu/defn to-clause :- ::lib.schema.expression/boolean"Convert an operator style parameter into an mbql clause. Will also do arity checks and throws an ex-info with`:type qp.error-type/invalid-parameter` if arity is incorrect."- [param]+ [param :- [:or ::lib.schema.parameter/parameter ::field-filter-operator-param]](let [{param-type :type, [a b :as param-value] :value, target :target, options :options} (normalize-param param)field-ref (or (match/match-one target[#{:field :expression} & _]diff --git a/src/metabase/query_processor/parameters/values.clj b/src/metabase/query_processor/parameters/values.cljindex 33fbd56b970..d6f5f133b9b 100644--- a/src/metabase/query_processor/parameters/values.clj+++ b/src/metabase/query_processor/parameters/values.clj@@ -73,7 +73,10 @@::params.types/no-value::single-value[:sequential ::single-value]- :map]])+ ::params.types/referenced-card-query+ ::params.types/referenced-table-query+ ::params.types/referenced-query-snippet+ ::params.types/temporal-unit]])(mu/defn- tag-targets"Given a template tag, returns a set of `target` structures that can be used to target the tag.@@ -366,7 +369,7 @@code that unquestioningly substituted any parameter passed in as a number directly into the SQL. This has long beenchanged for security purposes (avoiding SQL injection), but since users have come to expect comma-separated numericvalues to work we'll allow that (with validation) and return a vector to be converted to a list in the native query."- [value]+ [value :- [:or number? :string [:sequential [:or number? :string]]]](cond;; already parsed(number? value)@@ -384,7 +387,8 @@to parse it as appropriate based on the base type and semantic type of the Field associated with it). These arespecial cases for handling types that do not have an associated parameter type (such as `date` or `number`), such asUUID fields."- [effective-type :- ::lib.schema.common/base-type value]+ [effective-type :- ::lib.schema.common/base-type+ value :- :string](cond(isa? effective-type :type/UUID)(UUID/fromString value)@@ -419,7 +423,8 @@value.) For numbers, dates, and the like, this will parse the string appropriately; for `text` parameters, this willadditionally attempt handle special cases based on the base type of the Field, for example, parsing params for UUIDbase type Fields as UUIDs."- [param-type :- ::lib.schema.template-tag/type value]+ [param-type :- ::lib.schema.template-tag/type+ value :- ::parsed-param-value](cond(= value lib/parsed-param-no-value-placeholder)valuediff --git a/src/metabase/query_processor/pivot.clj b/src/metabase/query_processor/pivot.cljindex 28943885560..bf774859c5e 100644--- a/src/metabase/query_processor/pivot.clj+++ b/src/metabase/query_processor/pivot.clj@@ -21,7 +21,7 @@[metabase.lib.pivot :as lib.pivot][metabase.lib.schema :as lib.schema][metabase.lib.schema.aggregation :as lib.schema.aggregation]- [metabase.lib.schema.id :as lib.schema.id]+ [metabase.lib.schema.expression :as lib.schema.expression][metabase.lib.schema.info :as lib.schema.info][metabase.lib.util :as lib.util][metabase.models.visualization-settings :as mb.viz]@@ -61,13 +61,16 @@(mr/def ::pivot-cols [:sequential ::pivot.common/index])(mr/def ::pivot-measures [:sequential ::pivot.common/index])+(mr/def ::column-sort-order [:map-of [:maybe ::pivot.common/index] [:maybe :keyword]])+(mr/def ::pivot-opts [:maybe- [:map+ [:map {:closed true}[:pivot-rows {:optional true} [:maybe ::pivot-rows]][:pivot-cols {:optional true} [:maybe ::pivot-cols]][:pivot-measures {:optional true} [:maybe ::pivot-measures]][:show-row-totals {:optional true} [:maybe :boolean]]- [:show-column-totals {:optional true} [:maybe :boolean]]]])+ [:show-column-totals {:optional true} [:maybe :boolean]]+ [:column-sort-order {:optional true} [:maybe ::column-sort-order]]]])(mr/def ::pivot.common/breakout-combinations[:and@@ -181,7 +184,7 @@"Reduce the results of a single (sub)`query` using `rf` and initial value `init`."[query :- ::lib.schema/queryrf :- ::qp.schema/rf- init :- :any+ init :- ::qp.schema/accumulatorinfo :- [:maybe ::lib.schema.info/info]](if (qp.pipeline/canceled?)(ensure-reduced init)@@ -200,7 +203,7 @@(mu/defn- process-queries-append-results"Reduce the results of a sequence of `queries` using `rf` and initial value `init`."- [init+ [init :- ::qp.schema/accumulatorqueries :- [:maybe [:sequential ::lib.schema/query]]rf :- ::qp.schema/rfinfo :- [:maybe ::lib.schema.info/info]]@@ -314,9 +317,8 @@(mu/defn- column-name-pivot-options :- ::pivot-opts"Looks at the `pivot_table.column_split` key in the card's visualization settings and generates `pivot-rows` and`pivot-cols` to use for generating subqueries. Supports column name-based settings only."- [query :- [:map- [:database ::lib.schema.id/database]]- viz-settings :- [:maybe :map]]+ [query :- ::qp.schema/any-query+ viz-settings :- [:maybe :metabase.lib.schema.common/visualization-settings]](let [{:keys [rows columns values]} (:pivot_table.column_split viz-settings)show-row-totals (get viz-settings :pivot.show_row_totals true)show-column-totals (get viz-settings :pivot.show_column_totals true)@@ -343,12 +345,11 @@(when (some some? (vals pivot-opts))pivot-opts)))-(mu/defn- column-sort-order :- ::pivot-opts+(mu/defn- column-sort-order :- [:maybe ::column-sort-order]"Looks at the `pivot_table.column_sort_order` key in the card's visualization settings and generates a map from thecolumn's index to the setting (either ascending or descending)."- [query :- [:map- [:database ::lib.schema.id/database]]- viz-settings :- [:maybe :map]]+ [query :- ::qp.schema/any-query+ viz-settings :- [:maybe :metabase.lib.schema.common/visualization-settings]](let [metadata-provider (or (:lib/metadata query)(lib-be/application-database-metadata-provider (:database query)))query (lib/query metadata-provider query)@@ -367,9 +368,8 @@(mu/defn- field-ref-pivot-options :- ::pivot-opts"Looks at the `pivot_table.column_split` key in the card's visualization settings and generates `pivot-rows` and`pivot-cols` to use for generating subqueries. Supports field ref-based settings only."- [query :- [:map- [:database ::lib.schema.id/database]]- viz-settings :- [:maybe :map]]+ [query :- ::qp.schema/any-query+ viz-settings :- [:maybe :metabase.lib.schema.common/visualization-settings]](let [{:keys [rows columns values]} (:pivot_table.column_split viz-settings)show-row-totals (get viz-settings "pivot.show_row_totals" true)show-column-totals (get viz-settings "pivot.show_column_totals" true)@@ -414,9 +414,8 @@Field ref-based visualization settings are considered legacy and are not used for new questions. To not break existingquestions we need to support both old- and new-style settings until they are fully migrated."- [query :- [:map- [:database ::lib.schema.id/database]]- viz-settings :- [:maybe :map]]+ [query :- ::qp.schema/any-query+ viz-settings :- [:maybe :metabase.lib.schema.common/visualization-settings]](when viz-settings(let [{:keys [rows columns]} (:pivot_table.column_split viz-settings)](merge@@ -447,7 +446,7 @@against the last stage's breakouts in `query`. Returns nil when there is no `:pivot_table.column_split` or whenneither rows nor columns resolve."[query :- :metabase.lib.schema/query- viz-settings :- [:maybe :map]]+ viz-settings :- [:maybe :metabase.lib.schema.common/visualization-settings]](when-let [{:keys [rows columns]} (:pivot_table.column_split viz-settings)](let [row-uuids (resolve-refs-to-uuids query rows)col-uuids (resolve-refs-to-uuids query columns)]@@ -464,7 +463,7 @@Returns `query` unchanged when the last stage already has `:pivot`, when `viz-settings` is empty, or when no refsresolve."[query :- ::lib.schema/query- viz-settings :- [:maybe :map]]+ viz-settings :- [:maybe :metabase.lib.schema.common/visualization-settings]](let [clause (when (and (not (lib.pivot/has-pivot? query))(seq viz-settings))(build-pivot-clause query viz-settings))]@@ -540,7 +539,7 @@::qp.add-remaps/new-field-dimension-id]))))(mu/defn- remapped-indexes :- ::pivot.common/remapped-indexes- [breakouts]+ [breakouts :- [:maybe [:sequential ::lib.schema.expression/expression]]](let [remap-pairs (first(reduce (fn [[m i] breakout][(reduce-kv (fn [m remap-key id]@@ -811,7 +810,7 @@[[*on-parity-mismatch*]]. Parity checking is on by default in clojure.test tests.Wrap this call in [[metabase.query-processor.streaming/streaming-response]] yourself."- ([query]+ ([query :- ::qp.schema/any-query](run-pivot-query query nil))([query :- ::qp.schema/any-querydiff --git a/src/metabase/query_processor/pivot/middleware.clj b/src/metabase/query_processor/pivot/middleware.cljindex cd8307b60ec..2331bd59a88 100644--- a/src/metabase/query_processor/pivot/middleware.clj+++ b/src/metabase/query_processor/pivot/middleware.clj@@ -5,6 +5,7 @@(:require[medley.core :as m][metabase.lib.pivot :as lib.pivot]+ [metabase.lib.schema :as lib.schema][metabase.query-processor.pivot.common :as pivot.common][metabase.util.malli :as mu][metabase.util.malli.registry :as mr]@@ -14,7 +15,7 @@"Result-metadata-shape column entry spliced into `:cols` of pivot query results. Extends[[lib.pivot/pivot-grouping-column-metadata]] with the snake_case mirrors and explicit `:lib/desired-column-alias`the result-metadata layer expects."- (merge lib.pivot/pivot-grouping-column-metadata+ (merge (dissoc lib.pivot/pivot-grouping-column-metadata :lib/type :base-type){:display_name lib.pivot/pivot-grouping-column-name:lib/desired-column-alias lib.pivot/pivot-grouping-column-name:base_type :type/Integer@@ -69,9 +70,7 @@(mu/defn- column-mapping-for-subquery :- ::pivot-column-mapping[{num-remapped-cols :qp.pivot/num-remapped-colsnum-remapped-breakouts :qp.pivot/num-remapped-breakouts- :as _subquery} :- [:map- [:qp.pivot/num-remapped-cols nat-int?]- [:qp.pivot/num-remapped-breakouts ::pivot.common/num-breakouts]]+ :as _subquery} :- ::lib.schema/querysubquery-breakout-combination :- ::pivot.common/breakout-combination];; all pivot queries consist of *breakout columns* + *other columns*. Breakout columns are always first, and the only;; thing that can change between subqueries. The other columns will always be the same, and in the same order.@@ -118,9 +117,7 @@index is selected, we have to shift the indexes before which a mapped index is inserted."[{breakout-combination :qp.pivot/remapped-breakout-combinationremap :qp.pivot/remapped-indexes- :as _subquery} :- [:map- [:qp.pivot/remapped-breakout-combination ::pivot.common/breakout-combination]- [:qp.pivot/remapped-indexes ::pivot.common/remapped-indexes]]]+ :as _subquery} :- ::lib.schema/query](if (or (empty? remap)(empty? breakout-combination))breakout-combination@@ -142,7 +139,7 @@selected? (conj spliced-index)mapped-index (into (take-while some? (iterate remap mapped-index)))))))))))-(mu/defn- column-mapping [subquery :- :map]+(mu/defn- column-mapping [subquery :- ::lib.schema/query](let [full-breakout-combination (full-breakout-combination subquery)](column-mapping-for-subquery subquery full-breakout-combination)))@@ -158,7 +155,7 @@e.g.(f [1 2 3]) => [2 nil 3 nil 1]"- [subquery]+ [subquery :- ::lib.schema/query](perf/juxt* (for [mapping (column-mapping subquery)](if (nat-int? mapping)#(nth % mapping)diff --git a/src/metabase/query_processor/preprocess.clj b/src/metabase/query_processor/preprocess.cljindex fc91aae79eb..c62b87566bd 100644--- a/src/metabase/query_processor/preprocess.clj+++ b/src/metabase/query_processor/preprocess.clj@@ -119,7 +119,7 @@(mu/defn preprocess :- ::lib.schema/query"Fully preprocess a query, but do not compile it to a native query or execute it."- [query :- :map]+ [query :- ::qp.schema/any-query](when config/is-test?((requiring-resolve 'mb.hawk.init/assert-tests-are-not-initializing) "do not preprocess queries in top-level forms"))(qp.setup/with-qp-setup [query query]@@ -159,7 +159,7 @@"Return the `:cols` you would normally see in MBQL query results by preprocessing the query and calling `annotate` onit. This only works for pure MBQL queries, since it does not actually run the queries. Native queries or MBQLqueries with native source queries won't work, since we don't need the results."- [query :- :map]+ [query :- ::qp.schema/any-query](qp.setup/with-qp-setup [query query](let [preprocessed (-> query preprocess)];; TODO - we should throw an Exception if the query has a native source query with no attached metadata or atdiff --git a/src/metabase/query_processor/reducible.clj b/src/metabase/query_processor/reducible.cljindex 12c41d926c7..020a401042f 100644--- a/src/metabase/query_processor/reducible.clj+++ b/src/metabase/query_processor/reducible.clj@@ -14,7 +14,7 @@"Default function returning a reducing function. Results are returned in the 'standard' map format e.g.{:data {:cols [...], :rows [...]}, :row_count ...}"- [metadata]+ [metadata :- ::qp.schema/metadata](let [row-count (volatile! 0)rows (volatile! (transient []))](fn default-rf@@ -39,10 +39,11 @@`row-thunk` is a function that, when called, should return the next row in the results, or falsey if no more rowsexist."- ([row-thunk]+ ([row-thunk :- fn?](reducible-rows row-thunk qp.pipeline/*canceled-chan*))- ([row-thunk canceled-chan]+ ([row-thunk :- fn?+ canceled-chan :- [:maybe (lib.schema.common/instance-of-class clojure.core.async.impl.channels.ManyToManyChannel)]](reifyclojure.lang.IReduceInit(reduce [_ rf init]diff --git a/src/metabase/query_processor/schema.clj b/src/metabase/query_processor/schema.cljindex fd4aac38a84..1eab4a43a7c 100644--- a/src/metabase/query_processor/schema.clj+++ b/src/metabase/query_processor/schema.clj@@ -4,33 +4,104 @@;; because the QP still returns legacy-style metadata (for now)^{:clj-kondo/ignore [:discouraged-namespace]}[metabase.legacy-mbql.schema :as mbql.s]+ [metabase.lib-be.schema :as lib-be.schema]+ [metabase.lib.schema.common :as lib.schema.common][metabase.lib.schema.id :as lib.schema.id][metabase.util :as u][metabase.util.malli.registry :as mr][metabase.util.regex :as u.regex]))+(mr/def ::unnormalized-query+ "A legacy, MBQL 5, or internal query that has not been normalized yet, so its `:type` or `:lib/type` is still a string, e.g. as decoded from JSON."+ [:and+ [:map {:closed false, ::mr/deliberately-open true, :description "a query whose type key is still a string"}+ [:type {:optional true} [:enum "query" "native" "internal"]]+ [:lib/type {:optional true} [:= "mbql/query"]]]+ [:fn {:error/message "Query with a :type or :lib/type key"} (some-fn :type :lib/type)]])+(mr/def ::any-query"Schema for a map that is in the general shape of either a legacy MBQL or MBQL 5 query. Query may not be normalizedyet!This schema is not very strict because we need to handle different types of queries (legacy MBQL, MBQL 5,super-legacy MBQL, internal audit app queries, etc.) and it might not be normalized yet."- [:and- [:map- [:database {:optional true} [:or- ::lib.schema.id/database- ::lib.schema.id/saved-questions-virtual-database]]]- [:fn- {:error/message "Query with a :type or :lib/type key"}- (some-fn :type :lib/type)]- [:fn- {:error/message "Query should have :database unless it is :type :internal"}- #(or- (:database %)- (= (keyword (:type %)) :internal))]])--;; TODO -- fill this out a bit.-(mr/def ::metadata :any)+ [:or+ :metabase.lib.util/legacy-query+ :metabase.lib.util/mbql5-query+ ::lib-be.schema/internal-query+ ::unnormalized-query])++(mr/def ::internal-query.column+ "A column of an internal (audit app) query's results, as the query's function declares it."+ [:map {:closed true}+ [:name :string]+ [:display_name :string]+ [:base_type ::lib.schema.common/base-type]+ [:remapped_to {:optional true} :string]+ [:remapped_from {:optional true} :string]+ [:code {:optional true} :boolean]])++(mr/def ::insight.best-fit+ "A trendline formula fitted to a query's results, one of the curve shapes the insights calculation can fit."+ [:or+ [:tuple [:= :+] number? [:tuple [:= :*] number? [:= :x]]]+ [:tuple [:= :*] number? [:tuple [:= :exp] [:tuple [:= :*] number? [:= :x]]]]+ [:tuple [:= :+] number? [:tuple [:= :*] number? [:tuple [:= :log] [:= :x]]]]+ [:tuple [:= :*] number? [:tuple [:= :pow] [:= :x] number?]]])++(mr/def ::insight+ "One entry of the `:insights` calculated for a query's results."+ [:map {:closed true}+ [:last-value {:optional true} [:maybe number?]]+ [:previous-value {:optional true} [:maybe number?]]+ [:last-change {:optional true} [:maybe number?]]+ [:slope {:optional true} [:maybe number?]]+ [:offset {:optional true} [:maybe number?]]+ [:best-fit {:optional true} [:maybe ::insight.best-fit]]+ [:col {:optional true} [:maybe :string]]+ [:unit {:optional true} [:maybe :keyword]]])++(mr/def ::metadata+ "The map threaded through the post-processing `rff`/`rf` chain: an accumulator of query result metadata that grows+ as it passes through QP middleware, and is eventually merged into the final result's `:data` key. See+ [[metabase.query-processor.postprocess/middleware]] and [[metabase.query-processor.execute/middleware]] for the+ middleware that add to it."+ [:map {:closed true}+ [:cols {:optional true} [:sequential [:or+ ::result-metadata.column+ ::mbql.s/driver-column+ ::internal-query.column]]]+ [:native_form {:optional true} :metabase.query-processor.compile/compiled]+ [:dataset {:optional true} :boolean]+ [:model {:optional true} :boolean]+ [:viz-settings {:optional true} :metabase.lib.schema.common/visualization-settings]+ [:format-rows? {:optional true} :boolean]+ [:csv-include-bom? {:optional true} :boolean]+ [:results_timezone {:optional true} [:maybe :string]]+ [:requested_timezone {:optional true} [:maybe :string]]+ [:cache-version {:optional true} :int]+ [:last-ran {:optional true} (lib.schema.common/instance-of-class java.time.temporal.Temporal)]+ [:pivot-export-options {:optional true} [:map {:closed true}+ [:pivot-rows {:optional true} [:maybe [:sequential [:int {:min 0}]]]]+ [:pivot-cols {:optional true} [:maybe [:sequential [:int {:min 0}]]]]+ [:pivot-measures {:optional true} [:maybe [:sequential [:int {:min 0}]]]]+ [:show-row-totals {:optional true} :boolean]+ [:show-column-totals {:optional true} :boolean]+ [:column-sort-order {:optional true} [:maybe [:multi {:dispatch map?}+ [true [:map-of [:maybe [:int {:min 0}]] [:maybe :keyword]]]+ [false [:fn {:error/message "map"} map?]]]]]]]+ [:pivot? {:optional true} :boolean]+ [:is_sandboxed {:optional true} :boolean]+ [:download_perms {:optional true} [:or :keyword :string]]+ [:results_metadata {:optional true} [:map {:closed true}+ [:columns ::result-metadata.columns]]]+ [:insights {:optional true} [:maybe [:sequential ::insight]]]+ [:rows_truncated {:optional true} :int]+ [:pivot_rows_truncated {:optional true} :int]])++(mr/def ::accumulator+ "The running accumulator of a QP reducing function, whose shape is whatever that (possibly caller-supplied) reducing function accumulates."+ [:schema {::mr/deliberately-open true, :description "a reducing function's running accumulator"} :any])(mr/def ::rf"Schema for a reducing function."diff --git a/src/metabase/query_processor/setup.clj b/src/metabase/query_processor/setup.cljindex 34011cbfb72..bf93d7b3deb 100644--- a/src/metabase/query_processor/setup.clj+++ b/src/metabase/query_processor/setup.clj@@ -121,7 +121,7 @@{:query query, :type qp.error-type/invalid-query}))))))(mu/defn- do-with-resolved-database :- fn?- [f :- [:=> [:cat ::qp.schema/any-query] :any]]+ [f :- fn?](mu/fn[query :- ::qp.schema/any-query](let [query (set/rename-keys query {"database" :database})@@ -136,7 +136,7 @@(= (:lib/type query) :mbql/query) (assoc :lib/metadata (qp.store/metadata-provider))))(mu/defn- do-with-metadata-provider :- fn?- [f :- [:=> [:cat ::qp.schema/any-query] :any]]+ [f :- fn?](fn [query](cond(qp.store/initialized?)@@ -160,7 +160,7 @@(f (maybe-attach-metadata-provider-to-query query)))))))(mu/defn- do-with-driver :- fn?- [f :- [:=> [:cat ::qp.schema/any-query] :any]]+ [f :- fn?](fn [query](conddriver/*driver*@@ -185,7 +185,7 @@(f query))))))(mu/defn- do-with-database-local-settings :- fn?- [f :- [:=> [:cat ::qp.schema/any-query] :any]]+ [f :- fn?](fn [query](cond(setting/database-local-values)@@ -245,7 +245,7 @@TimeUnit/MILLISECONDS)))(mu/defn- do-with-canceled-chan :- fn?- [f :- [:=> [:cat ::qp.schema/any-query] :any]]+ [f :- fn?](fn [query](binding [qp.pipeline/*canceled-chan* (or qp.pipeline/*canceled-chan* (a/promise-chan))](let [timeout-task (schedule-query-timeout-cancel! qp.pipeline/*canceled-chan*)]@@ -282,7 +282,7 @@(mu/defn do-with-qp-setup"Impl for [[with-qp-setup]]."[query :- ::qp.schema/any-query- f :- [:=> [:cat ::qp.schema/any-query] :any]]+ f :- fn?];; TODO -- think about whether we should pre-compile this middleware(when (a.impl.dispatch/in-dispatch-thread?)(throw (ex-info "QP calls are not allowed inside core.async dispatch pool threads."diff --git a/src/metabase/query_processor/store.clj b/src/metabase/query_processor/store.cljindex b760800f3ae..080a86b1dd5 100644--- a/src/metabase/query_processor/store.clj+++ b/src/metabase/query_processor/store.clj@@ -46,6 +46,14 @@"Dynamic var used as the QP store for a given query execution."uninitialized-store)+(mr/def ::miscellaneous-value+ "Any value cached for the duration of a QP run via [[store-miscellaneous-value!]] or [[cached]]."+ [:schema {::mr/deliberately-open true, :description "a per-query-execution cached value of any kind"} :any])++(mr/def ::miscellaneous-value.ks+ "Key sequence for a value in the store's miscellaneous cache; [[cached]] prefixes it with a unique symbol."+ [:sequential [:or :keyword :symbol :int :string]])+(def ^:dynamic *DANGER-allow-replacing-metadata-provider*"This is (almost) only for tests! When enabled, [[with-metadata-provider]] can completely replace the current metadataprovider (and cache) with a new one. This is reset to false after the QP store is replaced the first time.@@ -69,8 +77,8 @@DEPRECATED -- use [[metabase.lib.metadata/general-cached-value]] going forward."{:deprecated "0.57.0"}- [ks :- [:sequential :any]- v]+ [ks :- ::miscellaneous-value.ks+ v :- ::miscellaneous-value](swap! *store* assoc-in ks v))(mu/defn miscellaneous-value@@ -78,11 +86,11 @@DEPRECATED -- use [[metabase.lib.metadata/general-cached-value]] going forward."{:deprecated "0.57.0"}- ([ks]+ ([ks :- ::miscellaneous-value.ks](miscellaneous-value ks nil))- ([ks :- [:sequential :any]- not-found]+ ([ks :- ::miscellaneous-value.ks+ not-found :- [:maybe [:= ::not-found]]](get-in @*store* ks not-found)))(defn cached-fn@@ -186,7 +194,7 @@(mu/defn do-with-metadata-provider"Implementation for [[with-metadata-provider]]."[database-id-or-metadata-providerable :- ::database-id-or-metadata-providerable- thunk :- [:=> [:cat] :any]]+ thunk :- fn?](cond(not (initialized?))(binding [*store* (atom {})]@@ -229,8 +237,7 @@(Note: it is preferable to use [[metabase.lib.core/lib-metadata-column->legacy-metadata-column]] instead of thisfunction if you REALLY need to do this sort of conversion.)"{:deprecated "0.48.0"}- [lib-metadata-col :- [:map- [:lib/type [:= :metadata/column]]]]+ [lib-metadata-col :- :metabase.lib.schema.metadata/column](-> lib-metadata-collib/lib-metadata-column->legacy-metadata-column(vary-meta assoc :type :metadata/column)))diff --git a/src/metabase/query_processor/timezone.clj b/src/metabase/query_processor/timezone.cljindex 74ce3410a20..cec978b6230 100644--- a/src/metabase/query_processor/timezone.clj+++ b/src/metabase/query_processor/timezone.clj@@ -106,15 +106,17 @@(^String [](results-timezone-id driver/*driver* ::db-from-store))- (^String [database]+ (^String [database :- ::database](results-timezone-id (:engine database) database))(^String [driver :- :keyworddatabase :- ::database& {:keys [use-report-timezone-id-if-unsupported?]- :or {use-report-timezone-id-if-unsupported? false}}]+ :or {use-report-timezone-id-if-unsupported? false}}+ :- [:maybe [:map {:closed true}+ [:use-report-timezone-id-if-unsupported? {:optional true} [:maybe :boolean]]]]](valid-timezone-id- (or *results-timezone-id-override*+ (or (valid-timezone-id *results-timezone-id-override*)(if use-report-timezone-id-if-unsupported?(valid-timezone-id (report-timezone-id*))(report-timezone-id-if-supported driver database))diff --git a/src/metabase/query_processor/util.clj b/src/metabase/query_processor/util.cljindex c5eb277a598..c70ec103a6d 100644--- a/src/metabase/query_processor/util.clj+++ b/src/metabase/query_processor/util.clj@@ -117,5 +117,5 @@(mu/defn internal-query? :- :boolean"Returns `true` if query is an internal query."- [{query-type :type} :- :map]+ [{query-type :type} :- ::qp.schema/any-query](= :internal (keyword query-type)))diff --git a/src/metabase/query_processor/util/add_alias_info.clj b/src/metabase/query_processor/util/add_alias_info.cljindex 5b5439062cf..30b7ff039b6 100644--- a/src/metabase/query_processor/util/add_alias_info.clj+++ b/src/metabase/query_processor/util/add_alias_info.clj@@ -86,18 +86,12 @@;;; of the kooky extra info we add here... none of it should affect calculated metadata(mu/defn- returned-columns :- :metabase.lib.metadata.calculation/returned-columns- [query :- [:merge- ::lib.schema/query- [:map- [::original ::lib.schema/query]]]+ [query :- ::lib.schema/querypath :- ::lib.walk/path](lib.walk/apply-f-for-stage-at-path lib/returned-columns (::original query) path))(mu/defn- resolve-field-ref :- :metabase.lib.metadata.calculation/visible-column- [query :- [:merge- ::lib.schema/query- [:map- [::original ::lib.schema/query]]]+ [query :- ::lib.schema/querystage-path :- ::lib.walk/pathfield-ref :- :mbql.clause/field](u/prog1 (lib.walk/apply-f-for-stage-at-path lib/metadata (::original query) stage-path field-ref)@@ -456,7 +450,7 @@(mr/def ::options[:map{:closed true}- [:globally-unique-join-aliases? {:default false} :any]])+ [:globally-unique-join-aliases? {:default false} :boolean]])(mu/defn- escape-join-aliases :- ::lib.schema/query[query :- ::lib.schema/query@@ -532,10 +526,10 @@If this is a nested column, the path to the column, e.g. for `grandparent.parent.child` this will be `[\"grandparent\"\"child\"]."- ([query]+ ([query :- [:or ::lib.schema/query :metabase.legacy-mbql.schema/Query :metabase.legacy-mbql.schema/MBQLInnerQuery]](add-alias-info query nil))- ([query :- :map+ ([query :- [:or ::lib.schema/query :metabase.legacy-mbql.schema/Query :metabase.legacy-mbql.schema/MBQLInnerQuery]options :- [:maybe ::options]](cond;; MBQL 5 querydiff --git a/src/metabase/query_processor/writeback.clj b/src/metabase/query_processor/writeback.cljindex 622377a79fe..5be5a6ea718 100644--- a/src/metabase/query_processor/writeback.clj+++ b/src/metabase/query_processor/writeback.clj@@ -77,7 +77,7 @@:string[:cat:string- [:* :any]]]]+ [:* :metabase.lib.schema.common/field-value]]]](let [mp (lib-be/application-database-metadata-provider db-id)[sql & params] (if (string? sql-or-sql+params)(cons sql-or-sql+params nil)diff --git a/src/metabase/request/cookies.clj b/src/metabase/request/cookies.cljindex 6e77d1dd3e6..3ed25386928 100644--- a/src/metabase/request/cookies.clj+++ b/src/metabase/request/cookies.clj@@ -3,13 +3,15 @@(:require[java-time.api :as t][metabase.config.core :as config]+ [metabase.request.schema :as request.schema][metabase.request.settings :as request.settings][metabase.request.util :as request.util]+ [metabase.session.schema][metabase.session.settings :as session.settings]- [metabase.util :as u][metabase.util.i18n :refer [tru]][metabase.util.log :as log][metabase.util.malli :as mu]+ [metabase.util.malli.schema :as ms][ring.util.response :as response]))(def metabase-session-cookie@@ -136,16 +138,14 @@(mu/defn set-session-cookies"Add the appropriate cookies to the `response` for the Session."- [request- response+ [request :- ::request.schema/request+ response :- ::request.schema/response{session-key :keysession-type :typeanti-csrf-token :anti_csrf_tokensession-expires-at :expires_at- :as _session-instance} :- [:map [:key [:or- uuid?- [:re u/uuid-regex]]]]- request-time]+ :as _session-instance} :- :metabase.session.schema/session+ request-time :- (ms/InstanceOfClass java.time.temporal.Temporal)](let [;; Calculate max-age based on session expiration if presentmax-age-seconds (when session-expires-at(let [expires-time (if (instance? java.time.OffsetDateTime session-expires-at)diff --git a/src/metabase/request/schema.clj b/src/metabase/request/schema.cljindex c1111143964..aa38c81ff48 100644--- a/src/metabase/request/schema.clj+++ b/src/metabase/request/schema.clj@@ -1,14 +1,136 @@(ns metabase.request.schema(:require- [metabase.util.malli.registry :as mr]))+ [metabase.server.streaming-response]+ [metabase.util.malli.registry :as mr]+ [metabase.util.malli.schema :as ms])+ (:import+ (metabase.server.streaming_response StreamingResponse)))++(comment metabase.server.streaming-response/keep-me);;; TODO (Cam 8/13/25) -- should this map be closed, that way we can make sure all the keys we might be using are;;; enumerated here?(mr/def ::current-user-info- [:map+ [:map {:closed true}[:metabase-user-id {:optional true} pos-int?][:is-superuser? {:optional true} :boolean][:is-data-analyst? {:optional true} :boolean][:user-locale {:optional true} [:maybe string?]][:is-group-manager? {:optional true} :boolean]- [:permissions-set {:optional true} [:set :string]]])+ [:permissions-set {:optional true} [:set :string]]+ [:auth-provider {:optional true} [:maybe :string]]+ [:settings {:optional true} [:maybe [:or ms/UserSettings :string]]]+ [:token-scopes {:optional true} [:maybe [:set [:or :keyword :string]]]]+ [:token-scopes-checked {:optional true} :boolean]])++(mr/def ::json-value+ "A JSON-shaped value: a scalar, a sequence of JSON values, or a string-keyed JSON object."+ [:or+ :string+ :keyword+ number?+ :boolean+ :nil+ [:sequential [:ref ::json-value]]+ [:map-of :string [:ref ::json-value]]])++(mr/def ::multipart-file+ "One `:multipart-params` entry for an uploaded file, as `ring.middleware.multipart-params` builds it."+ [:map {:closed true}+ [:filename :string]+ [:content-type :string]+ [:tempfile (ms/InstanceOfClass java.io.File)]+ [:size :int]])++(mr/def ::cookie-attrs+ "One `:cookies` entry of a request or response: a cookie's value and its attributes."+ [:map {:closed true}+ [:value {:optional true} :string]+ [:path {:optional true} :string]+ [:domain {:optional true} :string]+ [:max-age {:optional true} :int]+ [:expires {:optional true} :string]+ [:secure {:optional true} :boolean]+ [:http-only {:optional true} :boolean]+ [:same-site {:optional true} [:enum :strict :lax :none]]])++(mr/def ::request+ "A Ring request map, including the keys Metabase's own middleware stack adds."+ [:map {:closed true}+ [:server-port {:optional true} :int]+ [:server-name {:optional true} :string]+ [:remote-addr {:optional true} :string]+ [:uri {:optional true} :string]+ [:path-info {:optional true} :string]+ [:query-string {:optional true} [:maybe :string]]+ [:scheme {:optional true} :keyword]+ [:request-method {:optional true} :keyword]+ [:protocol {:optional true} :string]+ [:ssl-client-cert {:optional true} [:maybe (ms/InstanceOfClass java.security.cert.X509Certificate)]]+ [:headers {:optional true} [:map-of :string [:maybe :string]]]+ [:body {:optional true} [:maybe [:or (ms/InstanceOfClass java.io.InputStream) ms/RingRequestBody]]]+ [:query-params {:optional true} [:map-of :string [:maybe [:or :string [:sequential :string]]]]]+ [:form-params {:optional true} [:map-of :string [:or :string [:sequential :string]]]]+ [:multipart-params {:optional true} [:map-of :string [:or :string ::multipart-file]]]+ [:route-params {:optional true} ms/RingRequestParams]+ [:cookies {:optional true} [:map-of :string ::cookie-attrs]]+ [:route-metadata {:optional true} [:maybe :metabase.api.macros/route-metadata]]+ [:compojure/path {:optional true} :string]+ [:compojure/route-context {:optional true} [:maybe :string]]+ [:context {:optional true} [:maybe :string]]+ [:character-encoding {:optional true} [:maybe :string]]+ [:content-type {:optional true} [:maybe :string]]+ [:content-length {:optional true} [:maybe :int]]+ [:params {:optional true} ms/RingRequestParams]+ [:metabase-session-key {:optional true} [:maybe :string]]+ [:metabase-session-type {:optional true} [:maybe :keyword]]+ [:anti-csrf-token {:optional true} [:maybe :string]]+ [:metabase-user-id {:optional true} [:maybe :int]]+ [:metabase-request-id {:optional true} [:maybe :string]]+ [:request-id {:optional true} [:maybe [:or :uuid :string]]]+ [:browser-id {:optional true} [:maybe :string]]+ [:static-metabase-api-key {:optional true} [:maybe :string]]+ [:nonce {:optional true} [:maybe :string]]+ [:is-superuser? {:optional true} :boolean]+ [:is-data-analyst? {:optional true} :boolean]+ [:is-group-manager? {:optional true} :boolean]+ [:user-locale {:optional true} [:maybe :string]]+ [:embedding/auth-method {:optional true} [:maybe :string]]+ [:token-exchange? {:optional true} :boolean]+ [:metabase.server.middleware.offset-paging/limit {:optional true} [:maybe :int]]+ [:metabase.server.middleware.offset-paging/offset {:optional true} [:maybe :int]]+ [:accept {:optional true} [:maybe :string]]+ [:redirect-strategy {:optional true} [:maybe :keyword]]+ [:method {:optional true} [:maybe :keyword]]+ [:compojure/route {:optional true} [:maybe [:tuple :keyword :string]]]+ [:remember {:optional true} [:maybe :string]]+ [:slack/validated? {:optional true} :boolean]+ [:token-scopes {:optional true} [:maybe [:set [:or :keyword :string]]]]+ [:token-scopes-checked {:optional true} :boolean]+ [:mcp-ui-session-id {:optional true} [:maybe :string]]+ [:mcp-ui-credential {:optional true} [:maybe+ [:map {:closed true}+ [:v :int]+ [:uid :int]+ [:sid :string]+ [:exp :int]+ [:scp {:optional true} [:sequential :string]]+ [:unr {:optional true} :boolean]+ [:token-scopes {:optional true} [:maybe [:set [:or :string :keyword]]]]]]]])++(mr/def ::response+ "What an endpoint handler can return: JSON-shaped data, a full Ring response map, or a file/stream for downloads."+ [:or+ ms/RingResponseBody+ [:map {:closed true} [:id :string]]+ [:map {:closed true} [:success :boolean] [:session_id :string]]+ [:map {:closed true}+ [:status {:optional true} :int]+ [:headers {:optional true} [:map-of :string :string]]+ [:cookies {:optional true} [:map-of :string ::cookie-attrs]]+ [:body {:optional true} [:maybe [:or ms/RingResponseBody+ (ms/InstanceOfClass java.io.File)+ (ms/InstanceOfClass java.io.InputStream)]]]]+ (ms/InstanceOfClass java.io.File)+ (ms/InstanceOfClass java.io.InputStream)+ (ms/InstanceOfClass StreamingResponse)])diff --git a/src/metabase/request/session.clj b/src/metabase/request/session.cljindex 223ca261f5a..f4b3ca6403f 100644--- a/src/metabase/request/session.clj+++ b/src/metabase/request/session.clj@@ -26,14 +26,14 @@::none)(mu/defn- current-user-info->permissions-set :- [:maybe [:set :string]]- [{:keys [permissions-set metabase-user-id]} :- ::request.schema/current-user-info]+ [{:keys [permissions-set metabase-user-id]} :- [:or ::request.schema/current-user-info ::request.schema/request]](or permissions-set(some-> metabase-user-id perms/user-permissions-set)))(mu/defn do-with-current-user"Impl for [[with-current-user]] and [[metabase.server.middleware.session/with-current-user-for-request]]"- [{:keys [metabase-user-id is-superuser? is-data-analyst? user-locale settings is-group-manager?], :as current-user-info} :- [:maybe ::request.schema/current-user-info]- thunk]+ [{:keys [metabase-user-id is-superuser? is-data-analyst? user-locale settings is-group-manager?], :as current-user-info} :- [:maybe [:or ::request.schema/current-user-info ::request.schema/request]]+ thunk :- ifn?](binding [*current-user-id* metabase-user-idi18n/*user-locale* user-locale*is-group-manager?* (boolean is-group-manager?)diff --git a/src/metabase/request/util.clj b/src/metabase/request/util.cljindex 601feec235d..d84020276aa 100644--- a/src/metabase/request/util.clj+++ b/src/metabase/request/util.clj@@ -8,6 +8,7 @@[metabase.config.core :as config][metabase.embedding.util :as embed.util][metabase.request.current :as request.current]+ [metabase.request.schema :as request.schema][metabase.util :as u][metabase.util.i18n :refer [trs]][metabase.util.json :as json]@@ -109,7 +110,7 @@(mu/defn device-info :- DeviceInfo"Information about the device that made this request, as recorded by the `LoginHistory` table."- [{{:strs [user-agent]} :headers, :keys [browser-id token-exchange?], :as request}]+ [{{:strs [user-agent]} :headers, :keys [browser-id token-exchange?], :as request} :- ::request.schema/request](let [id (or browser-id(log/warn "Login request is missing device ID information"))description (or user-agentdiff --git a/src/metabase/revisions/api.clj b/src/metabase/revisions/api.cljindex a20087ac96c..9423358746c 100644--- a/src/metabase/revisions/api.clj+++ b/src/metabase/revisions/api.clj@@ -137,7 +137,7 @@table-id (assoc :table_id table-id))))))(when (contains? #{:model/Dashboard :model/Card} model)(collection/check-allowed-to-change-collection instance (:object revision))- (when (api/column-will-change? :dashboard_id instance (:object revision))+ (when (api/column-will-change? (:dashboard_id instance) (get (:object revision) :dashboard_id ::api/not-provided))(doseq [dashboard-id (keep identity [(:dashboard_id instance)(:dashboard_id (:object revision))])](api/write-check :model/Dashboard dashboard-id)))diff --git a/src/metabase/revisions/db.clj b/src/metabase/revisions/db.cljindex 6ae593fa8d7..925ede6fd42 100644--- a/src/metabase/revisions/db.clj+++ b/src/metabase/revisions/db.clj@@ -2,8 +2,10 @@"Application database queries for the revisions module. Every function here is a direct Toucan 2 call with noadditional logic, so no other namespace in the module runs a query itself (model definitions still use `toucan2.core`)."(:require+ [malli.core :as mc][metabase.lib.schema.id :as lib.schema.id][metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms][toucan2.core :as t2]))@@ -20,13 +22,38 @@id :- ms/PositiveInt](t2/select-one (t2/table-name model) :id id))+(def revisioned-model-row-schema+ "The literal registry keyword of the row/update schema of each model revisions are tracked for (a literal+ keyword, not a `require`, to avoid a dependency cycle with the module that owns each model)."+ {:model/Card :metabase.queries.schema/card.update+ :model/Dashboard :metabase.dashboards.schema/dashboard.update+ :model/Document :metabase.documents.schema/document.update+ :model/Exploration :metabase.explorations.schema/exploration.update+ :model/Measure :metabase.measures.schema/measure.update+ :model/Segment :metabase.segments.schema/segment.update+ :model/Transform :metabase.transforms.schema/transform.update})++(defn- revision-schema-key [prefix model]+ (keyword "metabase.revisions.db" (str prefix "." (name model))))++(doseq [[model schema] revisioned-model-row-schema]+ (mr/register! (revision-schema-key "revisioned-row" model)+ [:map {:closed true}+ [:model [:= model]]+ [:row [:merge schema [:map {:closed true} [:id {:optional true} ms/PositiveInt]]]]]))++(def ^:private RevisionedRow+ "A `{:model ..., :row ...}` pair naming one of the models revisions are tracked for, the row typed by that+ model's own row schema plus `:id` (a revisioned row is always a real, previously-selected row)."+ (into [:multi {:dispatch :model, :lazy-refs true}]+ (for [model (keys revisioned-model-row-schema)]+ [model (revision-schema-key "revisioned-row" model)])))+(mu/defn update-entity!- "Apply `changes` to the `model` row with `id`, returning the number updated. `changes` is a column map whose shape- can't be pinned to one model because `model` varies (see [[entity]])."- [model :- :keyword- id :- ms/PositiveInt- changes :- [:map-of :keyword [:maybe :some]]]- (t2/update! model id changes))+ "Apply `entity`'s `:row` (a column diff) to `entity`'s `:model` row with `id`, returning the number updated."+ [id :- ms/PositiveInt+ entity :- RevisionedRow]+ (t2/update! (:model entity) id (:row entity)))(mu/defn parameter-card-ids"The Card ids of the ParameterCards of the `parameterized-object-type` with `parameterized-object-id`."@@ -156,12 +183,70 @@revision-id :- ms/PositiveInt](t2/select-one-fn :object :model/Revision :model model-name :model_id model-id :id revision-id))+(def ^:private revision-object-extra-keys+ "Extra keys [[metabase.revisions.impl.dashboard/serialize-instance]] and friends add to some models' revision+ `:object` beyond their own row schema."+ {:model/Dashboard [[:cards {:optional true} [:sequential [:merge :metabase.dashboards.schema/dashboard-card.update+ [:map {:closed true}+ [:id {:optional true} ::lib.schema.id/dashcard]+ [:series {:optional true} [:sequential ::lib.schema.id/card]]]]]]+ [:tabs {:optional true} [:sequential [:merge :metabase.dashboards.schema/dashboard-tab.update+ [:map {:closed true} [:id {:optional true} ms/PositiveInt]]]]]]})++(def revisioned-model-select-schema+ "The literal registry keyword of the row schema (as selected, hydrated keys included) of each model revisions are+ tracked for."+ {:model/Card :metabase.queries.schema/card+ :model/Dashboard :metabase.dashboards.schema/dashboard+ :model/Document :metabase.documents.schema/document+ :model/Exploration :metabase.explorations.schema/exploration+ :model/Measure :metabase.measures.schema/measure+ :model/Segment :metabase.segments.schema/segment+ :model/Transform :metabase.transforms.schema/transform})++(mr/def ::unregistered-model-object+ "The revision `:object` of a model outside [[revisioned-model-row-schema]] (a test double), whose keys that model's own `serialize-instance` owns."+ [:map {:closed false, ::mr/deliberately-open true}])++(mr/def ::stored-revision-object+ "A revision `:object` as stored, whose keys the Metabase version that recorded it owns (fields may since have been dropped)."+ [:map {:closed false, ::mr/deliberately-open true}])++(doseq [[model schema] revisioned-model-select-schema]+ (mr/register! (revision-schema-key "revision-row" model)+ [:map {:closed true}+ [:model [:= (name model)]]+ [:model_id ms/PositiveInt]+ [:user_id ::lib.schema.id/user]+ [:object [:merge schema+ (into [:map {:closed true} [:id {:optional true} ms/PositiveInt]]+ (get revision-object-extra-keys model))]]+ [:is_creation :boolean]+ [:is_reversion :boolean]+ [:message {:optional true} [:maybe :string]]]))+(def ^:private RevisionRow+ "A Revision row, `:object` typed by the row schema of the model named `:model` (a string, e.g. \"Card\"), plus+ `:id` (a revisioned object is always a real, previously-selected row)."+ (conj (into [:multi {:dispatch :model, :lazy-refs true}]+ (map (fn [model] [(name model) (revision-schema-key "revision-row" model)]))+ (keys revisioned-model-row-schema))+ [::mc/default [:map {:closed true}+ [:model :string]+ [:model_id ms/PositiveInt]+ [:user_id ::lib.schema.id/user]+ [:object ::unregistered-model-object]+ [:is_creation :boolean]+ [:is_reversion :boolean]+ [:message {:optional true} [:maybe :string]]]]))++(def ^:private RevertedRevisionRow+ "A Revision row recording a revert to the stored `:object` of an earlier Revision."[:map {:closed true}- [:model [:or :keyword :string]]+ [:model :string][:model_id ms/PositiveInt][:user_id ::lib.schema.id/user]- [:object :map]+ [:object ::stored-revision-object][:is_creation :boolean][:is_reversion :boolean][:message {:optional true} [:maybe :string]]])@@ -173,7 +258,7 @@(mu/defn insert-revision-returning!"Insert the Revision `row` and return the inserted instance."- [row :- RevisionRow]+ [row :- RevertedRevisionRow](t2/insert-returning-instance! :model/Revision row))(mu/defn latest-editors-reduciblediff --git a/src/metabase/revisions/models/revision.clj b/src/metabase/revisions/models/revision.cljindex 664029c4d14..65516ba3f8f 100644--- a/src/metabase/revisions/models/revision.clj+++ b/src/metabase/revisions/models/revision.clj@@ -1,11 +1,16 @@(ns metabase.revisions.models.revision(:require[clojure.data :as data]+ [malli.core :as mc][metabase.config.core :as config]+ [metabase.explorations.schema]+ [metabase.measures.schema][metabase.models.interface :as mi][metabase.queries.core :as queries][metabase.revisions.db :as revisions.db][metabase.revisions.models.revision.diff :refer [diff-strings*]]+ [metabase.segments.schema]+ [metabase.transforms.schema][metabase.util :as u][metabase.util.i18n :refer [deferred-tru tru]][metabase.util.json :as json]@@ -43,7 +48,7 @@;; an error if a field in an earlier version has since been dropped, but is still present in the revision.;; This is best effort — other kinds of schema changes could still break the ability to revert successfully.revert-instance (select-keys serialized-instance valid-columns)]- (revisions.db/update-entity! model id revert-instance)))+ (revisions.db/update-entity! id {:model model :row revert-instance})))(defmulti diff-map"Return a map describing the difference between `object-1` and `object-2`."@@ -191,18 +196,30 @@(recur (conj acc (add-revision-details model r1 r2))(conj more r2))))))+(def ^:private PushRevisionInput+ (into [:multi {:dispatch :entity}]+ (conj (vec (for [[model schema] revisions.db/revisioned-model-select-schema]+ [model [:map {:closed true}+ [:id pos-int?]+ [:object schema]+ [:entity [:= model]]+ [:user-id pos-int?]+ [:is-creation? {:optional true} [:maybe :boolean]]+ [:message {:optional true} [:maybe :string]]]]))+ [::mc/default [:map {:closed true}+ [:id pos-int?]+ [:object ::revisions.db/unregistered-model-object]+ [:entity [:fn toucan-model?]]+ [:user-id pos-int?]+ [:is-creation? {:optional true} [:maybe :boolean]]+ [:message {:optional true} [:maybe :string]]]])))+(mu/defn push-revision!"Record a new Revision for `entity` with `id` if it's changed compared to the last revision.Returns `object` or `nil` if the object does not changed."[{:keys [id entity user-id objectis-creation? message]- :or {is-creation? false}} :- [:map {:closed true}- [:id pos-int?]- [:object :map]- [:entity [:fn toucan-model?]]- [:user-id pos-int?]- [:is-creation? {:optional true} [:maybe :boolean]]- [:message {:optional true} [:maybe :string]]]]+ :or {is-creation? false}} :- PushRevisionInput](let [entity-name (name entity)serialized-object (serialize-instance entity id (dissoc object :message))last-object (revisions.db/latest-revision-object entity-name id)diff --git a/src/metabase/revisions/models/revision/last_edit.clj b/src/metabase/revisions/models/revision/last_edit.cljindex cf4d00bc175..732c3aa930d 100644--- a/src/metabase/revisions/models/revision/last_edit.clj+++ b/src/metabase/revisions/models/revision/last_edit.clj@@ -11,10 +11,12 @@[clojure.set :as set][java-time.api :as t][medley.core :as m]+ [metabase.dashboards.schema][metabase.revisions.db :as revisions.db][metabase.util.malli :as mu][metabase.util.malli.schema :as ms]- [steffan-westcott.clj-otel.api.trace.span :as span]))+ [steffan-westcott.clj-otel.api.trace.span :as span]+ [toucan2.core :as t2]))(def ^:private model->db-model {:card "Card" :dashboard "Dashboard"})@@ -38,7 +40,9 @@"Add the last edited information to a card. Will add a key `:last-edit-info`. Model should be one of `:dashboard` or`:card`. Gets the last edited information from the revisions table. If you need this information from a put route,use `@api/*current-user*` and a current timestamp since revisions are events and asynchronous."- [items+ [items :- [:sequential [:multi {:dispatch t2/model}+ [:model/Card :metabase.queries.schema/card]+ [:model/Dashboard :metabase.dashboards.schema/dashboard]]]model :- [:enum :dashboard :card]](let [ids (into #{} (map :id) items)](span/with-span!@@ -61,7 +65,7 @@the revisions table. But this table is populated from events asynchronously so when editing and wantinglast-edit-info, you must construct it from `@api/*current-user*` and the current timestamp rather than checking therevisions table as those revisions may not be present yet."- [user]+ [user :- [:maybe :metabase.users.schema/user]](merge {:timestamp (t/instant)}(select-keys user [:id :first_name :last_name :email])))@@ -72,13 +76,18 @@[:card {:optional true} [:map-of :int LastEditInfo]][:dashboard {:optional true} [:map-of :int LastEditInfo]]])+(def ^:private FetchLastEditedInfoArgs+ [:map {:closed true}+ [:card-ids {:optional true} [:maybe [:sequential ms/PositiveInt]]]+ [:dashboard-ids {:optional true} [:maybe [:sequential ms/PositiveInt]]]])+(mu/defn fetch-last-edited-info :- [:maybe CollectionLastEditInfo]"Fetch edited info from the revisions table. Revision information is timestamp, user id, email, first and lastname. Takes card-ids and dashboard-ids and returns a map structured like{:card {card_id {:id :email :first_name :last_name :timestamp}}:dashboard {dashboard_id {:id :email :first_name :last_name :timestamp}}}"- [{:keys [card-ids dashboard-ids]}]+ [{:keys [card-ids dashboard-ids]} :- FetchLastEditedInfoArgs](when (seq (concat card-ids dashboard-ids))(let [latest-changes (revisions.db/latest-changes card-ids dashboard-ids)](->> latest-changesdiff --git a/src/metabase/revisions/schema.clj b/src/metabase/revisions/schema.cljindex cc1fd8850cf..dd18db09dff 100644--- a/src/metabase/revisions/schema.clj+++ b/src/metabase/revisions/schema.clj@@ -11,18 +11,10 @@(mr/def ::revision"A Revision as selected from the app DB: every column of `:revision`."- [:map {:closed true}- [:id ms/PositiveInt]- [:model [:or :keyword :string]]- [:model_id [:maybe :int]]- [:user_id ::lib.schema.id/user]- [:timestamp ms/TemporalInstant]- [:object ::revision.object]- [:is_reversion :boolean]- [:is_creation :boolean]- [:message [:maybe :string]]- [:most_recent :boolean]- [:metabase_version [:maybe :string]]])+ [:merge+ ::revision.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::revision.update"What an update (or insert) of a Revision accepts: every column of `:revision` except `id`, all optional."diff --git a/src/metabase/run_tracking/db.clj b/src/metabase/run_tracking/db.cljindex f8394b89aad..861a8453c0a 100644--- a/src/metabase/run_tracking/db.clj+++ b/src/metabase/run_tracking/db.clj@@ -20,7 +20,7 @@(def ^:private StaleSpec"A single staleness cutoff: a `model` row is stale under this spec when `column` is older than `age``unit`s ago."- [:map+ [:map {:closed true}[:column :keyword][:age pos-int?][:unit StaleUnit]])@@ -78,7 +78,12 @@[model :- :keywordactive :- ActivePredicateids :- [:sequential ms/PositiveInt]- terminal :- [:map-of :keyword [:maybe [:or :string :keyword :boolean :int]]]]+ terminal :- [:map {:closed true}+ [:status {:optional true} [:maybe [:or :string :keyword]]]+ [:end_time {:optional true} [:maybe [:or :string :keyword]]]+ [:ended_at {:optional true} [:maybe [:or :string :keyword]]]+ [:is_active {:optional true} [:maybe :boolean]]+ [:message {:optional true} [:maybe :string]]]](t2/query {:update (t2/table-name model):set terminal:where [:and (active-clause active) [:in :id ids]]}))diff --git a/src/metabase/search/api.clj b/src/metabase/search/api.cljindex 47e387a9cf9..4edfa5320c2 100644--- a/src/metabase/search/api.clj+++ b/src/metabase/search/api.clj@@ -143,7 +143,7 @@(mu/defn- set-weights![context :- :keyword- overrides :- [:map-of keyword? double?]]+ overrides :- [:map-of ::search.config/scorer-key double?]](api/check-superuser)(when (= context :all)(throw (ex-info "Cannot set weights for all context"diff --git a/src/metabase/search/appdb/core.clj b/src/metabase/search/appdb/core.cljindex 0d7f9988cf7..7917e372451 100644--- a/src/metabase/search/appdb/core.clj+++ b/src/metabase/search/appdb/core.clj@@ -75,7 +75,7 @@(defn- view-count-percentiles*[p-value](into {} (for [{:keys [model vcp]} (search.db/view-count-percentile-rows (search.index/active-table) p-value)]- [(keyword model) vcp])))+ [model vcp])))(def ^{:private true:arglists '([p-value])}diff --git a/src/metabase/search/config.clj b/src/metabase/search/config.cljindex f42559f49cb..4cf2eaa1ad1 100644--- a/src/metabase/search/config.clj+++ b/src/metabase/search/config.clj@@ -171,6 +171,17 @@(concat (keys static-default-weights)(mapcat keys (vals static-context-weights)))))+(defn- scorer-key? [k]+ (and (keyword? k) (contains? known-rankers (if (namespace k) (keyword (namespace k)) k))))++(mr/def ::scorer-key+ "A weights map key: a scorer name, optionally namespaced with a scorer-specific parameter (e.g.+ `:model/dashboard`), that collapses to one of [[known-rankers]]."+ [:fn scorer-key?])++(mr/def ::weights+ [:map-of ::scorer-key number?])+(def ^:private FilterDef"A relaxed definition, capturing how we can write the filter - with some fields omitted."[:map {:closed true}@@ -193,6 +204,33 @@[:required-feature [:maybe :keyword]][:engine [:maybe :keyword]]])+(def ^:private FilterDefNoKey+ "[[FilterDef]] without the `:key` entry, which [[build-filters]] fills in from the map key."+ [:map {:closed true}+ [:type :keyword]+ [:field {:optional true} :string]+ [:context-key {:optional true} :keyword]+ [:supported-value? {:optional true} ifn?]+ [:required-feature {:optional true} :keyword]+ [:engine {:optional true} :keyword]])++(def ^:private FilterDefs+ "The filter-name -> (relaxed) filter-definition map [[build-filters]] accepts, keyed by every filter name in+ [[filters]]."+ [:map {:closed true}+ [:archived {:optional true} FilterDefNoKey]+ [:collection-id {:optional true} FilterDefNoKey]+ [:created-at {:optional true} FilterDefNoKey]+ [:creator-id {:optional true} FilterDefNoKey]+ [:database-id {:optional true} FilterDefNoKey]+ [:id {:optional true} FilterDefNoKey]+ [:last-edited-at {:optional true} FilterDefNoKey]+ [:last-editor-id {:optional true} FilterDefNoKey]+ [:native-query {:optional true} FilterDefNoKey]+ [:verified {:optional true} FilterDefNoKey]+ [:curated {:optional true} FilterDefNoKey]+ [:display-type {:optional true} FilterDefNoKey]])+(mu/defn- build-filter :- Filter[{k :key t :type :keys [context-key field supported-value? required-feature engine]} :- FilterDef]{:key k@@ -204,7 +242,7 @@:engine (or engine :all)})(mu/defn- build-filters :- [:map-of :keyword Filter]- [m]+ [m :- FilterDefs](-> (reduce #(assoc-in %1 [%2 :key] %2) m (keys m))(update-vals build-filter)))@@ -444,7 +482,7 @@;; deliberately not exposed over HTTP)[:vector-search-force-index? {:optional true} [:maybe :boolean]][:search-string {:optional true} [:maybe ms/NonBlankString]]- [:weights {:optional true} [:maybe [:map-of :keyword number?]]]+ [:weights {:optional true} [:maybe ::weights]];;;; optional;;diff --git a/src/metabase/search/db.clj b/src/metabase/search/db.cljindex 22a382a5caf..a702047c954 100644--- a/src/metabase/search/db.clj+++ b/src/metabase/search/db.clj@@ -13,10 +13,23 @@[metabase.search.in-place.legacy :as legacy][metabase.search.ingestion.query :as ingestion.query][metabase.search.schema :as search.schema]+ [metabase.search.spec :as search.spec]+ [metabase.util.honey-sql-2 :as h2x][metabase.util.malli :as mu][metabase.util.malli.schema :as ms][toucan2.core :as t2]))+(def ^:private SearchIndexRow+ "A row of the search index table: `search.spec/attr-columns` (with `:id`/`:created_at`/`:updated_at` renamed the+ way `metabase.search.appdb.index/document->entry` renames them), the base columns, and the extra columns the+ active engine specialization adds, each value a Honey SQL 2 expression."+ (into [:map {:closed true}]+ (for [column (into #{:model :display_data :legacy_input :model_id :model_created_at :model_updated_at+ :updated_at :search_vector :with_native_query_vector :search_terms+ :native_search_terms}+ (remove #{:id :created_at :updated_at} search.spec/attr-columns))]+ [column {:optional true} ::h2x/expr])))+(mu/defn spec-index-reducible-rows"A reducible of the indexable rows of `search-model` (see `metabase.search.ingestion.query/spec-index-query`)matching `where-clause`, or every row when it is nil.@@ -58,7 +71,7 @@[index-table :- [:or :keyword :string]search-ctx :- search.config/SearchContextsearch-string :- [:maybe :string]- view-count-percentiles :- [:map-of :keyword [:maybe number?]]]+ view-count-percentiles :- [:map-of (into [:enum] search.spec/search-models) [:maybe [:or number? :string]]]](t2/query (search.scoring/with-scores search-ctx(search.scoring/scorers search-ctx view-count-percentiles)(appdb.query/base-filtered-query index-table search-ctx search-string [:legacy_input]))))@@ -132,7 +145,7 @@"Upsert `entries` into the search index `table`, on conflict of `(model, model_id)` overwriting every other columnwith the new value."[table :- [:or :keyword :string]- entries :- [:sequential :map]]+ entries :- [:sequential SearchIndexRow]](when (seq entries)(let [update-keys (vec (disj (set (mapcat keys entries)) :id :model :model_id))excluded-kw (fn [column] (keyword (str "excluded." (name column))))]@@ -185,7 +198,7 @@(mu/defn insert-rows!"Insert `entries` into the search index `table`."[table :- [:or :keyword :string]- entries :- [:sequential :map]]+ entries :- [:sequential SearchIndexRow]](t2/insert! table entries))(mu/defn index-entry-countdiff --git a/src/metabase/search/debug.clj b/src/metabase/search/debug.cljindex 6dede7d8927..c256ef288d3 100644--- a/src/metabase/search/debug.clj+++ b/src/metabase/search/debug.clj@@ -2,6 +2,7 @@"Diagnostics for the search debug API: explain why a given entity does *not* appear in a search query's results.See [[diagnose]]."(:require+ [metabase.search.config :refer [SearchContext]][metabase.search.engine :as search.engine][metabase.search.impl :as search.impl][metabase.search.ingestion :as search.ingestion]@@ -70,7 +71,7 @@`:filtered`/`:not-permitted`, `:not-matching`, `:candidate`) come from [[metabase.search.engine/diagnose]]; theengine-independent stages are decided here. `:details` always carries the resolved engine. `:not-searchable`reflects current spec/DB truth, so it wins even if a stale index row lingers."- [search-ctx :- :map+ [search-ctx :- SearchContextexpected-model :- ms/NonBlankStringexpected-id :- pos-int?](let [engine-details {:resolved-engine (:search-engine search-ctx)diff --git a/src/metabase/search/impl.clj b/src/metabase/search/impl.cljindex 331eb091d5d..dc9806f3929 100644--- a/src/metabase/search/impl.clj+++ b/src/metabase/search/impl.clj@@ -307,7 +307,7 @@[:include-dashboard-questions? {:optional true} [:maybe boolean?]][:include-metadata? {:optional true} [:maybe boolean?]][:display-type {:optional true} [:maybe [:set ms/NonBlankString]]]- [:weights {:optional true} [:maybe [:map-of :keyword number?]]]])+ [:weights {:optional true} [:maybe ::search.config/weights]]])(mu/defn search-context :- SearchContext"Create a new search context that you can pass to other functions like [[search]]."diff --git a/src/metabase/search/in_place/filter.clj b/src/metabase/search/in_place/filter.cljindex 211b30a0fa7..c8b1fd0d64a 100644--- a/src/metabase/search/in_place/filter.clj+++ b/src/metabase/search/in_place/filter.clj@@ -32,6 +32,27 @@(def ^:private true-clause [:= [:inline 1] [:inline 1]])(def ^:private false-clause [:= [:inline 0] [:inline 1]])+(def ^:private HoneySQLQuery+ "A partially-built Honey SQL query map for the legacy (index-free) search query. Mirrors+ `metabase.search.in-place.legacy/HoneySQLQuery` (duplicated here to avoid a circular require)."+ [:schema+ {:registry+ {::expr [:or :keyword :string number? :boolean nil?+ [:sequential [:ref ::expr]]+ [:set [:ref ::expr]]+ [:ref ::query]]+ ::query [:map {:closed true}+ [:select {:optional true} [:sequential [:ref ::expr]]]+ [:from {:optional true} [:sequential [:ref ::expr]]]+ [:where {:optional true} [:ref ::expr]]+ [:with {:optional true} [:sequential [:ref ::expr]]]+ [:join {:optional true} [:sequential [:ref ::expr]]]+ [:left-join {:optional true} [:sequential [:ref ::expr]]]+ [:union-all {:optional true} [:sequential [:ref ::expr]]]+ [:order-by {:optional true} [:sequential [:ref ::expr]]]+ [:limit {:optional true} [:ref ::expr]]]}}+ [:ref ::query]])+(def ^:private max-document-search-length"Cap the number of characters of a document's prose-mirror body that the legacy engine LIKE-scans.A leading-wildcard LIKE can't use an index, so this bounds the worst case for pathologically large@@ -413,7 +434,7 @@(mu/defn build-filters :- :map"Build the search filters for a model."- [honeysql-query :- :map+ [honeysql-query :- HoneySQLQuerymodel :- SearchableModelsearch-context :- SearchContext](let [{:keys [modelsdiff --git a/src/metabase/search/in_place/legacy.clj b/src/metabase/search/in_place/legacy.cljindex b31b002f77b..9ae8ba568e9 100644--- a/src/metabase/search/in_place/legacy.clj+++ b/src/metabase/search/in_place/legacy.clj@@ -23,10 +23,36 @@[metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay][toucan2.core :as t2]))+(def ^:private honeysql-registry+ "Registry backing [[HoneySQLExpr]] and [[HoneySQLQuery]]: `::expr` is a column/table keyword, a literal, a+ (possibly nested) operator clause, or a subquery."+ {::expr [:or :keyword :string number? :boolean nil? (ms/InstanceOfClass java.time.temporal.Temporal)+ [:sequential [:ref ::expr]]+ [:set [:ref ::expr]]+ [:ref ::query]]+ ::query [:map {:closed true}+ [:select {:optional true} [:or [:ref ::expr] [:sequential [:ref ::expr]]]]+ [:from {:optional true} [:or [:ref ::expr] [:sequential [:ref ::expr]]]]+ [:where {:optional true} [:ref ::expr]]+ [:with {:optional true} [:or [:ref ::expr] [:sequential [:ref ::expr]]]]+ [:join {:optional true} [:or [:ref ::expr] [:sequential [:ref ::expr]]]]+ [:left-join {:optional true} [:or [:ref ::expr] [:sequential [:ref ::expr]]]]+ [:inner-join {:optional true} [:or [:ref ::expr] [:sequential [:ref ::expr]]]]+ [:union-all {:optional true} [:or [:ref ::expr] [:sequential [:ref ::expr]]]]+ [:order-by {:optional true} [:or [:ref ::expr] [:sequential [:ref ::expr]]]]+ [:limit {:optional true} [:ref ::expr]]]})++(def ^:private HoneySQLExpr+ [:schema {:registry honeysql-registry} [:ref ::expr]])+(def ^:private HoneySQLColumn[:or:keyword- [:tuple :any :keyword]])+ [:tuple HoneySQLExpr :keyword]])++(def HoneySQLQuery+ "A partially-built Honey SQL query map for the legacy (index-free) search query."+ [:schema {:registry honeysql-registry} [:ref ::query]])(defmethod search.engine/supported-engine? :search.engine/in-place [_]true)@@ -115,11 +141,15 @@:data_authority :text:data_layer :text))+(def ^:private SearchColumn+ "Enum of every key of [[all-search-columns]]."+ (into [:enum] (keys all-search-columns)))+(mu/defn- canonical-columns :- [:sequential HoneySQLColumn]"Returns a seq of lists of canonical columns for the search query with the given `model` Will return column namesprefixed with the `model` name so that it can be used in criteria. Projects a `nil` for columns the `model` doesn'thave and doesn't modify aliases."- [model :- SearchableModel, col-alias->honeysql-clause :- [:map-of :keyword HoneySQLColumn]]+ [model :- SearchableModel, col-alias->honeysql-clause :- [:map-of SearchColumn HoneySQLColumn]](for [[search-col col-type] all-search-columns:let [maybe-aliased-col (get col-alias->honeysql-clause search-col)]](cond@@ -152,7 +182,7 @@(mu/defn- add-table-db-id-clause"Add a WHERE clause to only return tables with the given DB id.Used in data picker for joins because we can't join across DB's."- [query :- :map id :- [:maybe ms/PositiveInt]]+ [query :- HoneySQLQuery id :- [:maybe ms/PositiveInt]](if (some? id)(sql.helpers/where query [:= id :db_id])query))@@ -160,7 +190,7 @@(mu/defn- add-card-db-id-clause"Add a WHERE clause to only return cards with the given DB id.Used in data picker for joins because we can't join across DB's."- [query :- :map id :- [:maybe ms/PositiveInt]]+ [query :- HoneySQLQuery id :- [:maybe ms/PositiveInt]](if (some? id)(sql.helpers/where query [:= id :database_id])query))@@ -186,7 +216,7 @@(mu/defn add-collection-join-and-where-clauses"Add a `WHERE` clause to the query to only return Collections the Current User has access to; join against Collection,so we can return its `:name`."- [honeysql-query :- :map+ [honeysql-query :- HoneySQLQuerymodel :- [:maybe :string]search-ctx :- SearchContext](let [collection-id-col (case model@@ -215,7 +245,7 @@and some of them are dummy column casted to the correct type.This function then will replace the dummy column with alias is `target-alias` with the `with` column."- [query :- :map+ [query :- HoneySQLQuerytarget-alias :- :keywordwith :- :keyword](let [selects (:select query)@@ -232,7 +262,7 @@:with with})))))(mu/defn- with-last-editing-info :- :map- [query :- :map+ [query :- HoneySQLQuerymodel :- [:enum "card" "dashboard"]](-> query(replace-select :last_editor_id :r.user_id)@@ -243,7 +273,7 @@[:= :r.model (search-model/search-model->revision-model model)]])))(mu/defn- with-moderated-status :- :map- [query :- :map+ [query :- HoneySQLQuerymodel :- [:enum "card" "dataset" "dashboard"]](-> query(replace-select :moderated_status :mr.status)@@ -325,10 +355,8 @@[:collection.type :collection_type][:collection.location :collection_location][:collection.authority_level :collection_authority_level]- [:dashboard.name :dashboard_name]:dashboard_idbookmark-col dashboardcard-count-col- :result_metadata[:display :display_type]))(defmethod columns-for-model "document"diff --git a/src/metabase/search/schema.clj b/src/metabase/search/schema.cljindex 7c1b3f27201..447391a1460 100644--- a/src/metabase/search/schema.clj+++ b/src/metabase/search/schema.clj@@ -6,15 +6,10 @@(mr/def ::search-index-metadata"A SearchIndexMetadata as selected from the app DB: every column of `:search_index_metadata`."- [:map {:closed true}- [:id ms/PositiveInt]- [:engine [:or :keyword :string]]- [:version :string]- [:index_name :string]- [:status [:maybe [:or :keyword :string]]]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]- [:lang_code :string]])+ [:merge+ ::search-index-metadata.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::search-index-metadata.update"What an update (or insert) of a SearchIndexMetadata accepts: every column of `:search_index_metadata` except `id`, all optional."diff --git a/src/metabase/secrets/models/secret.clj b/src/metabase/secrets/models/secret.cljindex 2fcd3308812..5f08ba28187 100644--- a/src/metabase/secrets/models/secret.clj+++ b/src/metabase/secrets/models/secret.clj@@ -330,7 +330,9 @@Fetches the stored secret and fills in `-path` `-options` `-value` for each secret property.Operates on `:details`, `:write_data_details`, and `:admin_details`."[database]- (let [driver (driver.u/database->driver database)+ (let [driver (if-let [engine (:engine database)]+ (keyword engine)+ (driver.u/database->driver (:id database)))hydrate (fn [details](reduce-over-details-secret-values driver details hydrate-redacted-secret))];; Very low-level operation here, so not using driver.conn/* utils:@@ -355,7 +357,7 @@"Ensures that all possible secret property values are removed from `:details`, `:write_data_details`, and`:admin_details`. This is a transformation on `:model/Database` `results-transform`."[database]- (let [clean #(clean-secret-properties-from-details % (driver.u/database->driver database))]+ (let [clean #(clean-secret-properties-from-details % (keyword (:engine database)))];; Very low-level operation here, so not using driver.conn/* utils:(-> database(m/update-existing :details clean)@@ -369,7 +371,7 @@(if-let [details (get database details-key)](let [original-details (get (t2/original database) details-key)updated-details (reduce-over-details-secret-values- (driver.u/database->driver database)+ (keyword (:engine database))details(fn [db-details conn-prop-nm conn-prop](let [kws (->possible-secret-property-names conn-prop-nm)diff --git a/src/metabase/secrets/schema.clj b/src/metabase/secrets/schema.cljindex d09a7812788..6044550e347 100644--- a/src/metabase/secrets/schema.clj+++ b/src/metabase/secrets/schema.clj@@ -7,16 +7,10 @@(mr/def ::secret"A Secret as selected from the app DB: every column of `:secret`."- [:map {:closed true}- [:id ms/PositiveInt]- [:version :int]- [:creator_id [:maybe ::lib.schema.id/user]]- [:created_at ms/TemporalInstant]- [:updated_at [:maybe ms/TemporalInstant]]- [:name :string]- [:kind [:or :keyword :string]]- [:source [:maybe [:or :keyword :string]]]- [:value [:or bytes? :string]]])+ [:merge+ ::secret.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::secret.update"What an update (or insert) of a Secret accepts: every column of `:secret` except `id`, all optional."diff --git a/src/metabase/security_center/schema.clj b/src/metabase/security_center/schema.cljindex b010196388c..c06353e82fe 100644--- a/src/metabase/security_center/schema.clj+++ b/src/metabase/security_center/schema.clj@@ -1,6 +1,7 @@(ns metabase.security-center.schema"Malli schemas for the Security Center module."(:require+ [metabase.util.honey-sql-2 :as h2x][metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms]))@@ -18,7 +19,7 @@(mr/def ::version-range"A single affected version range with inclusive min and exclusive fixed."- [:map+ [:map {:closed true}[:min ::semver][:fixed ::semver]])@@ -27,7 +28,7 @@(mr/def ::download-jar-url"A downloadable JAR for a given fixed version."- [:map+ [:map {:closed true}[:version ::semver][:url :string]])@@ -39,36 +40,40 @@for the app DB in use."[:enum :default :h2 :mysql :postgres])+(mr/def ::honeysql-query+ "A HoneySQL 2 query map, as EDN keyed by its top-level clause keywords."+ [:map {:closed true}+ [:with {:optional true} [:sequential [:tuple :keyword [:ref ::honeysql-query]]]]+ [:select {:optional true} [:sequential ::h2x/expr]]+ [:delete-from {:optional true} ::h2x/expr]+ [:from {:optional true} [:sequential [:or ::h2x/expr [:ref ::honeysql-query]]]]+ [:join {:optional true} [:sequential [:or ::h2x/expr [:ref ::honeysql-query]]]]+ [:where {:optional true} ::h2x/expr]+ [:group-by {:optional true} [:sequential ::h2x/expr]]+ [:having {:optional true} ::h2x/expr]+ [:order-by {:optional true} [:sequential ::h2x/expr]]+ [:limit {:optional true} ::h2x/expr]+ [:offset {:optional true} ::h2x/expr]])+(mr/def ::matching-query"HoneySQL query keyed by dialect. nil means affects all instances.Stored as EDN to preserve keywords that HoneySQL requires for identifiers/operators."- [:maybe [:map-of ::driver ms/OpaqueJSONObject]])+ [:maybe [:map {:closed true}+ [:default {:optional true} ::honeysql-query]+ [:h2 {:optional true} ::honeysql-query]+ [:mysql {:optional true} ::honeysql-query]+ [:postgres {:optional true} ::honeysql-query]]])(mr/def ::security-advisory.matching-query"The `:matching_query` column of a SecurityAdvisory, decoded."- :map)+ ::matching-query)(mr/def ::security-advisory"A SecurityAdvisory as selected from the app DB: every column of `:security_advisory`."- [:map {:closed true}- [:id ms/PositiveInt]- [:advisory_id :string]- [:severity [:or :keyword :string]]- [:title :string]- [:description :string]- [:advisory_url [:maybe :string]]- [:remediation :string]- [:affected_versions ::affected-versions]- [:matching_query [:maybe ::security-advisory.matching-query]]- [:published_at ms/TemporalInstant]- [:fetched_at ms/TemporalInstant]- [:match_status [:or :keyword :string]]- [:last_evaluated_at [:maybe ms/TemporalInstant]]- [:acknowledged_by [:maybe :int]]- [:acknowledged_at [:maybe ms/TemporalInstant]]- [:last_notified_at [:maybe ms/TemporalInstant]]- [:updated_at ms/TemporalInstant]- [:download_jar_urls [:maybe ::download-jar-urls]]])+ [:merge+ ::security-advisory.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::security-advisory.update"What an update (or insert) of a SecurityAdvisory accepts: every column of `:security_advisory` except `id`, all optional."diff --git a/src/metabase/segments/api.clj b/src/metabase/segments/api.cljindex 5733e6cf1df..4fcee6e7424 100644--- a/src/metabase/segments/api.clj+++ b/src/metabase/segments/api.clj@@ -41,7 +41,8 @@the create-check runs wherever a Segment is authored."[{:keys [name description definition], :as body}];; TODO - why can't we set other properties like `show_in_getting_started` when we create the Segment?- (let [table-id (definition-table-id definition)]+ (let [table-id (definition-table-id definition)+ definition (lib-be/normalize-query definition)](api/create-check :model/Segment (assoc body :table_id table-id))(let [segment (api/check-500(segments.db/insert-segment! table-id api/*current-user-id* name description definition))]@@ -108,7 +109,8 @@(when (not= new-table-id (:table_id existing))(api/create-check :model/Segment {:table_id new-table-id}))))(when changes- (segments.db/update-segment! id changes))+ (segments.db/update-segment! id (cond-> changes+ (:definition changes) (update :definition lib-be/normalize-query))))(u/prog1 (hydrated-segment id)(events/publish-event! :event/segment-update{:object <> :user-id api/*current-user-id* :revision-message revision_message}))))diff --git a/src/metabase/segments/db.clj b/src/metabase/segments/db.cljindex c8181f67740..2df37e708a8 100644--- a/src/metabase/segments/db.clj+++ b/src/metabase/segments/db.clj@@ -46,7 +46,7 @@(mu/defn table"The Table with `table-id`, or nil."- [table-id :- ::lib.schema.id/table]+ [table-id :- [:maybe ::lib.schema.id/table]](t2/select-one :model/Table :id table-id {:from [(warehouse-schema-overlay/table-query)]}))(mu/defn table-database-iddiff --git a/src/metabase/segments/schema.clj b/src/metabase/segments/schema.cljindex 23ee5c5be07..1973f128132 100644--- a/src/metabase/segments/schema.clj+++ b/src/metabase/segments/schema.clj@@ -3,9 +3,11 @@[metabase.lib-be.schema :as lib-be.schema][metabase.lib.core :as lib][metabase.lib.schema.id :as lib.schema.id]+ [metabase.users.schema][metabase.util.i18n :refer [deferred-tru]][metabase.util.malli.registry :as mr]- [metabase.util.malli.schema :as ms]))+ [metabase.util.malli.schema :as ms]+ [metabase.warehouse-schema.schema]))(mr/def ::definition"Schema for a segment's `:definition`; accepts a full MBQL query, converting legacy MBQL on decode."@@ -28,24 +30,17 @@(mr/def ::segment.definition"The `:definition` column of a Segment, decoded."- :map)+ ::lib-be.schema/maybe-legacy-query)(mr/def ::segment- "A Segment as selected from the app DB: every column of `:segment`."- [:map {:closed true}- [:id ms/PositiveInt]- [:table_id ::lib.schema.id/table]- [:creator_id ::lib.schema.id/user]- [:name :string]- [:description [:maybe :string]]- [:archived :boolean]- [:definition [:maybe ::segment.definition]]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]- [:points_of_interest [:maybe :string]]- [:caveats [:maybe :string]]- [:show_in_getting_started :boolean]- [:entity_id :string]])+ "A Segment as selected from the app DB: every column of `:segment`, plus `:creator` and `:table` some callers+ hydrate onto it."+ [:merge+ ::segment.update+ [:map {:closed true}+ [:id ms/PositiveInt]+ [:creator {:optional true} [:maybe :metabase.users.schema/user]]+ [:table {:optional true} [:maybe :metabase.warehouse-schema.schema/table]]]])(mr/def ::segment.update"What an update (or insert) of a Segment accepts: every column of `:segment` except `id`, all optional."diff --git a/src/metabase/server/instance.clj b/src/metabase/server/instance.cljindex ec42c40e2ee..6ea8ac7c810 100644--- a/src/metabase/server/instance.clj+++ b/src/metabase/server/instance.clj@@ -104,7 +104,25 @@"Create a new async Jetty server with `handler` and `options`. Handy for creating the real Metabase web server, andcreating one-off web servers for tests and REPL usage."^Server [handler :- ::api.macros/handler- options :- [:maybe :map]]+ options :- [:maybe [:map {:closed true}+ [:port {:optional true} :int]+ [:host {:optional true} :string]+ [:max-threads {:optional true} :int]+ [:min-threads {:optional true} :int]+ [:max-queued {:optional true} :int]+ [:max-idle-time {:optional true} :int]+ [:send-server-version? {:optional true} :boolean]+ [:request-header-size {:optional true} :int]+ [:daemon? {:optional true} :boolean]+ [:ssl? {:optional true} :boolean]+ [:ssl-port {:optional true} :int]+ [:keystore {:optional true} :string]+ [:key-password {:optional true} :string]+ [:truststore {:optional true} :string]+ [:trust-password {:optional true} :string]+ [:client-auth {:optional true} :keyword]+ [:sni-host-check? {:optional true} :boolean]+ [:join? {:optional true} :boolean]]]];; if any API endpoint functions aren't at the very least returning a channel to fetch the results later after 10;; minutes we're in serious trouble. (Almost everything 'slow' should be returning a channel before then, but;; some things like CSV downloads don't currently return channels at this time)diff --git a/src/metabase/server/middleware/browser_cookie.clj b/src/metabase/server/middleware/browser_cookie.cljindex d57fae82824..fa9162836e4 100644--- a/src/metabase/server/middleware/browser_cookie.clj+++ b/src/metabase/server/middleware/browser_cookie.clj@@ -6,6 +6,7 @@(:require[java-time.api :as t][metabase.request.core :as request]+ [metabase.request.schema :as request.schema][metabase.util.malli :as mu][metabase.util.malli.schema :as ms][ring.util.response :as response]))@@ -30,7 +31,10 @@{:same-site :none, :secure true}{:same-site :lax})))-(mu/defn- add-browser-id-cookie [request response browser-id :- ms/NonBlankString]+(mu/defn- add-browser-id-cookie+ [request :- ::request.schema/request+ response :- ::request.schema/response+ browser-id :- ms/NonBlankString](response/set-cookie response browser-id-cookie-name browser-id (cookie-options request)))(defn ensure-browser-id-cookiediff --git a/src/metabase/server/middleware/session.clj b/src/metabase/server/middleware/session.cljindex 7e53966d486..3d259e3d9a1 100644--- a/src/metabase/server/middleware/session.clj+++ b/src/metabase/server/middleware/session.clj@@ -104,7 +104,8 @@(mu/defn- current-user-info-for-session :- [:maybe ::request.schema/current-user-info]"Return User ID and superuser status for Session with `session-key` if it is valid and not expired."- [session-key anti-csrf-token]+ [session-key :- [:maybe :string]+ anti-csrf-token :- [:maybe :string]](when (and session-key (valid-session-key? session-key) (init-status/complete?))(some-> (server.db/session-user-info (session/hash-session-key session-key)anti-csrf-token@@ -188,7 +189,7 @@shape returned by the session/api-key resolvers and additionally attaches `:token-scopes`, so themerged request carries both the user identity and the access the token was granted. This is theonly place an OAuth access token authenticates a request to the general (`/api/*`) API."- [request]+ [request :- ::request.schema/request](when (init-status/complete?)(when-let [token (oauth-server/extract-bearer-token request)](when-let [{:keys [user-id scopes]} (oauth-server/resolve-access-token token)]diff --git a/src/metabase/session/api.clj b/src/metabase/session/api.cljindex 88de91c452b..59d602e28d9 100644--- a/src/metabase/session/api.clj+++ b/src/metabase/session/api.clj@@ -77,7 +77,9 @@(mu/defn- ldap-login :- [:maybe [:or session.schema/SessionSchema [:map [:mfa/pending? [:= true]]]]]"If LDAP is enabled and a matching user exists return a new Session for them (or an MFA-pendingresult map when a second factor is required), or `nil` if they couldn't be authenticated."- [username password device-info :- request/DeviceInfo]+ [username :- ms/NonBlankString+ password :- ms/NonBlankString+ device-info :- request/DeviceInfo](when (sso/ldap-enabled)(let [result (auth-identity/login! :provider/ldap{:username usernamediff --git a/src/metabase/session/schema.clj b/src/metabase/session/schema.cljindex f6a75c3f743..2ab1ed26b81 100644--- a/src/metabase/session/schema.clj+++ b/src/metabase/session/schema.clj@@ -14,15 +14,13 @@(mr/def ::session"A Session as selected from the app DB: every column of `:core_session`."- [:map {:closed true}- [:id :string]- [:user_id ::lib.schema.id/user]- [:created_at ms/TemporalInstant]- [:anti_csrf_token [:maybe :string]]- [:key_hashed :string]- [:auth_identity_id [:maybe ms/PositiveInt]]- [:expires_at [:maybe ms/TemporalInstant]]- [:last_active_at [:maybe ms/TemporalInstant]]])+ [:merge+ ::session.update+ [:map {:closed true}+ [:id :string]+ [:key {:optional true} :string]+ [:type {:optional true} [:enum :normal :full-app-embed]]+ [:mfa_auth_identity_id {:optional true} [:maybe ms/PositiveInt]]]])(mr/def ::session.update"What an update (or insert) of a Session accepts: every column of `:core_session` except `id`, all optional."diff --git a/src/metabase/settings/models/setting.clj b/src/metabase/settings/models/setting.cljindex dfa4c81e512..123e2324c3e 100644--- a/src/metabase/settings/models/setting.clj+++ b/src/metabase/settings/models/setting.clj@@ -103,7 +103,7 @@(get-value-of-type :string (keyword id)))(defmethod serdes/load-one! "Setting" [{:keys [key value]} _]- (set-value-of-type! :string key value))+ (set-value-of-type! :string key (some-> value str)))(def ^:private Type[:fn@@ -973,8 +973,11 @@(str writer))](first (str/split-lines s)))+ (nil? value)+ nil+:else- value))+ (str value)))(defmethod set-value-of-type! :csv[_setting-type setting-definition-or-name new-value]diff --git a/src/metabase/settings/schema.clj b/src/metabase/settings/schema.cljindex ea078e44ad6..b7cdd3d2c7e 100644--- a/src/metabase/settings/schema.clj+++ b/src/metabase/settings/schema.clj@@ -5,10 +5,9 @@(mr/def ::setting"A Setting as selected from the app DB: every column of `:setting`."- [:map {:closed true}- [:key :string]- [:value :string]- [:value_with_aad [:maybe :string]]])+ [:merge+ ::setting.update+ [:map {:closed true}]])(mr/def ::setting.update"What an update (or insert) of a Setting accepts: every column of `:setting` except `id`, all optional."diff --git a/src/metabase/source_swap/mbql.clj b/src/metabase/source_swap/mbql.cljindex 6fcee461e90..7cacc8f99a7 100644--- a/src/metabase/source_swap/mbql.clj+++ b/src/metabase/source_swap/mbql.clj@@ -6,6 +6,7 @@[metabase.lib.options :as lib.options][metabase.lib.ref :as lib.ref][metabase.lib.schema :as lib.schema]+ [metabase.lib.schema.expression :as lib.schema.expression][metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.join :as lib.schema.join][metabase.lib.schema.metadata :as lib.schema.metadata]@@ -33,7 +34,7 @@(mu/defn- walk-clause-field-refs :- :any"Walks a clause and applies a function to all `:field` clauses."- [clause :- :any+ [clause :- [:or ::lib.schema.expression/expression :metabase.lib.schema.order-by/order-by]f :- fn?](lib.walk/walk-clause clause(fn [clause]@@ -66,8 +67,8 @@"Upgrade all field refs in a list of clauses to use name-based field refs when possible."[query :- ::lib.schema/querystage-number :- :int- clauses :- [:sequential :any]- {:keys [distinct?]} :- [:map [:distinct? :boolean]]]+ clauses :- [:sequential [:or ::lib.schema.expression/expression :metabase.lib.schema.order-by/order-by]]+ {:keys [distinct?]} :- [:map {:closed true} [:distinct? :boolean]]](into [](cond-> (map (fn [clause](walk-clause-field-refs clause #(upgrade-field-ref query stage-number %))))@@ -231,8 +232,8 @@[query :- ::lib.schema/querystage-number :- :intfield-id-mapping :- ::field-id-mapping- clauses :- [:sequential :any]- {:keys [distinct?]} :- [:map [:distinct? :boolean]]]+ clauses :- [:sequential [:or ::lib.schema.expression/expression :metabase.lib.schema.order-by/order-by]]+ {:keys [distinct?]} :- [:map {:closed true} [:distinct? :boolean]]](into [](cond-> (map (fn [clause](walk-clause-field-refs clause #(swap-field-ref query stage-number field-id-mapping %))))diff --git a/src/metabase/source_swap/schema.clj b/src/metabase/source_swap/schema.cljindex c33eb382198..9b141c0dc3b 100644--- a/src/metabase/source_swap/schema.clj+++ b/src/metabase/source_swap/schema.clj@@ -23,7 +23,7 @@:foreign-key-mismatch])(mr/def ::column-mapping- [:map+ [:map {:closed true}[:source {:optional true} ::lib.schema.metadata/column][:target {:optional true} ::lib.schema.metadata/column][:errors {:optional true} [:sequential ::column-error]]])diff --git a/src/metabase/source_swap/util.clj b/src/metabase/source_swap/util.cljindex ee480f76d95..716add48353 100644--- a/src/metabase/source_swap/util.clj+++ b/src/metabase/source_swap/util.clj@@ -11,7 +11,7 @@(mu/defn column-match-key :- :string"Gets the column match key from a column. This is used to match columns by name or alias."- [column :- [:map [:name :string]]]+ [column :- ::lib.schema.metadata/lib-or-legacy-column](or (:lib/desired-column-alias column) (:name column)))(mu/defn source-columns :- [:sequential ::lib.schema.metadata/column]diff --git a/src/metabase/sql_tools/common.clj b/src/metabase/sql_tools/common.cljindex 36f7040bc29..a053bc0156a 100644--- a/src/metabase/sql_tools/common.clj+++ b/src/metabase/sql_tools/common.clj@@ -61,10 +61,13 @@(mu/defn table-name :- [:maybe :string]"Computes a table name from a table reference"- [raw-col :- [:map+ [raw-col :- [:map {:closed true}[:database {:optional true} :string][:schema {:optional true} :string]- [:table {:optional true} :string]]]+ [:table {:optional true} :string]+ [:table-alias {:optional true} :string]+ [:column {:optional true} :string]+ [:type {:optional true} :keyword]]](when (:table raw-col)(->> [:database :schema :table](keep raw-col)diff --git a/src/metabase/sql_tools/core.clj b/src/metabase/sql_tools/core.cljindex ebe05b98722..64da3a1860f 100644--- a/src/metabase/sql_tools/core.clj+++ b/src/metabase/sql_tools/core.clj@@ -22,14 +22,14 @@(mr/def ::table-spec"Schema for table identifier used in replacements. `:db` is the top-level qualifier fordrivers that emit one (BigQuery project, ClickHouse database)."- [:map+ [:map {:closed true}[:table :string][:schema {:optional true} [:maybe :string]][:db {:optional true} [:maybe :string]]])(mr/def ::column-spec"Schema for column identifier used in replacements."- [:map+ [:map {:closed true}[:column :string][:table {:optional true} [:maybe :string]][:schema {:optional true} [:maybe :string]]@@ -39,21 +39,21 @@"Schema for table replacement target - can be a string or a map with schema/table/db."[:or:string- [:map+ [:map {:closed true}[:schema {:optional true} [:maybe :string]][:table {:optional true} [:maybe :string]][:db {:optional true} [:maybe :string]]]])(mr/def ::replacements"Schema for replace-names replacements map."- [:map+ [:map {:closed true}[:schemas {:optional true} [:map-of :string :string]][:tables {:optional true} [:map-of ::table-spec ::table-replacement-value]][:columns {:optional true} [:map-of ::column-spec :string]]])(mr/def ::replace-names-opts"Schema for replace-names options."- [:map+ [:map {:closed true}[:allow-unused? {:optional true} :boolean]])(mr/def ::simple-query-result@@ -83,7 +83,7 @@(mu/defn returned-columns :- [:sequential :map]"Return appdb columns for the `native-query`."[driver :- :keyword- native-query]+ native-query :- :metabase.lib.schema/native-only-query](let [parser (sql-tools.settings/current-parser-backend)](metrics/with-operation-timing [parser "returned-columns"](interface/returned-columns-impl parser driver native-query))))@@ -91,7 +91,7 @@(mu/defn referenced-tables :- [:set :map]"Return tables referenced by the `native-query`"[driver :- :keyword- native-query]+ native-query :- :metabase.lib.schema/native-only-query](let [parser (sql-tools.settings/current-parser-backend)](metrics/with-operation-timing [parser "referenced-tables"](interface/referenced-tables-impl parser driver native-query))))@@ -102,7 +102,7 @@This includes fields in SELECT, WHERE, JOIN ON, GROUP BY, ORDER BY, etc.Returns a set of :metadata/column maps."[driver :- :keyword- native-query]+ native-query :- :metabase.lib.schema/native-only-query](let [parser (sql-tools.settings/current-parser-backend)](metrics/with-operation-timing [parser "referenced-fields"](interface/referenced-fields-impl parser driver native-query))))@@ -123,7 +123,7 @@(mu/defn validate-query :- [:set :map]"Validate native query. Returns a set of validation errors (empty set if valid)."[driver :- :keyword- native-query]+ native-query :- :metabase.lib.schema/native-only-query](let [parser (sql-tools.settings/current-parser-backend)](metrics/with-operation-timing [parser "validate-query"](interface/validate-query-impl parser driver native-query))))@@ -145,7 +145,9 @@without sanitization. Callers MUST ensure replacement values are system-generated(e.g., database metadata). Never passuser-controlled input as replacement values."- ([driver sql-string replacements]+ ([driver :- [:maybe :keyword]+ sql-string :- :string+ replacements :- ::replacements](replace-names driver sql-string replacements {}))([driver :- [:maybe :keyword]sql-string :- :string@@ -158,7 +160,7 @@(mu/defn referenced-tables-raw :- [:sequential ::table-spec]"Given a driver and sql string, returns a sequence of {:schema <name> :table <name>} maps."[driver :- :keyword- sql-str :- :string]+ sql-str :- [:maybe :string]](let [parser (sql-tools.settings/current-parser-backend)](metrics/with-operation-timing [parser "referenced-tables-raw"](interface/referenced-tables-raw-impl parser driver sql-str))))diff --git a/src/metabase/sso/ldap/default_implementation.clj b/src/metabase/sso/ldap/default_implementation.cljindex 955a9dac047..da92e78d854 100644--- a/src/metabase/sso/ldap/default_implementation.clj+++ b/src/metabase/sso/ldap/default_implementation.clj@@ -25,7 +25,7 @@(def LDAPSettings"Options passed to LDAP integration implementations. These are just the various LDAP Settings from[[metabase.sso.ldap]], packaged up as a single map so implementations don't need to fetch Setting values directly."- [:map+ [:map {:closed true}[:first-name-attribute ms/NonBlankString][:last-name-attribute ms/NonBlankString][:email-attribute ms/NonBlankString]@@ -57,7 +57,7 @@user-baseoptions)](log/debugf "LDAP search returned %d result(s)" (count search-result))- (some-> (first search-result) u/lower-case-map-keys))))+ (some-> (first search-result) (update-keys (comp u/lower-case-en name))))))(mu/defn- process-group-membership-filter :- ms/NonBlankString"Replace DN and UID placeholders with values returned by the LDAP server."@@ -87,16 +87,16 @@(mu/defn ldap-search-result->user-info :- [:maybe UserInfo]"Convert the result "[ldap-connection :- (ms/InstanceOfClass LDAPConnectionPool)- {:keys [dn uid], :as result} :- :map+ {:strs [dn uid], :as result} :- (ms/string-keyed-map [:or :string [:sequential :string] [:set :string]]){:keys [first-name-attributelast-name-attributeemail-attributesync-groups?]:as settings} :- LDAPSettingsgroup-membership-filter :- ms/NonBlankString]- (let [{first-name (keyword first-name-attribute)- last-name (keyword last-name-attribute)- email (keyword email-attribute)} result]+ (let [first-name (get result (u/lower-case-en first-name-attribute))+ last-name (get result (u/lower-case-en last-name-attribute))+ email (get result (u/lower-case-en email-attribute))]{:dn dn:first-name first-name:last-name last-name@@ -104,7 +104,7 @@:groups (when sync-groups?;; Active Directory and others (like FreeIPA) will supply a `memberOf` overlay attribute for;; groups. Otherwise we have to make the inverse query to get them.- (or (u/one-or-many (:memberof result))+ (or (u/one-or-many (get result "memberof"))(user-groups ldap-connection dn uid settings group-membership-filter)[]))}))@@ -122,7 +122,7 @@(mu/defn ldap-groups->mb-group-ids :- [:set ms/PositiveInt]"Translate a set of a user's group DNs to a set of MB group IDs using the configured mappings."[ldap-groups :- [:maybe [:sequential ms/NonBlankString]]- {:keys [group-mappings]} :- [:select-keys LDAPSettings [:group-mappings]]]+ {:keys [group-mappings]} :- LDAPSettings](-> group-mappings(select-keys (map #(DN. (str %)) ldap-groups))vals@@ -131,7 +131,7 @@(mu/defn all-mapped-group-ids :- [:set ms/PositiveInt]"Returns the set of all MB group IDs that have configured mappings."- [{:keys [group-mappings]} :- [:select-keys LDAPSettings [:group-mappings]]]+ [{:keys [group-mappings]} :- LDAPSettings](-> group-mappingsvalsflattendiff --git a/src/metabase/staleness/core.clj b/src/metabase/staleness/core.cljindex feea80b3c73..f370df44bdf 100644--- a/src/metabase/staleness/core.clj+++ b/src/metabase/staleness/core.clj@@ -4,7 +4,35 @@explicit set of models and call this multimethod per model.Kept dependency-light (no model requires) so model namespaces can require it without creating a- load cycle.")+ load cycle."+ (:require+ [metabase.util.honey-sql-2 :as h2x]+ [metabase.util.malli.registry :as mr]))++(mr/def ::predicate+ "A HoneySQL boolean expression as used by [[find-stale-query]]'s `:where`/`:left-join` clauses: a bare+ column keyword, a scalar value, or an operator keyword applied to nested predicates/values."+ [:or+ :keyword+ :string+ number?+ boolean?+ :time/local-date+ h2x/Literal+ [:set [:maybe :int]]+ [:cat :keyword [:* [:maybe [:ref ::predicate]]]]])++(mr/def ::select-entry+ [:or :keyword [:tuple [:or :keyword h2x/Literal] :keyword]])++(mr/def ::query+ "The shape [[find-stale-query]] must return, and [[metabase-enterprise.stale.db]] consumes: a HoneySQL+ map selecting `id`/`model`/`name`/`last_used_at` from a single table."+ [:map {:closed true}+ [:select [:sequential ::select-entry]]+ [:from :keyword]+ [:left-join {:optional true} [:sequential [:or :keyword ::predicate]]]+ [:where {:optional true} ::predicate]])(defmulti find-stale-query"Return a HoneySQL map selecting stale candidates for `model`:diff --git a/src/metabase/sync/analyze/classify.clj b/src/metabase/sync/analyze/classify.cljindex f648020ebd1..93d1408d07f 100644--- a/src/metabase/sync/analyze/classify.clj+++ b/src/metabase/sync/analyze/classify.clj@@ -69,10 +69,16 @@:model/Table (sync.db/update-table! (u/the-id original-model) values-to-set))true)))+(def ^:private ClassifyOpts+ [:map {:closed true}+ [:fields-failed :int]+ [:exists-name :boolean]])+(mu/defn- classify!"Run various classifiers on `field` and its `fingerprint`, and save any detected changes.Returns updated `field`"- ([field :- i/FieldInstance opts]+ ([field :- i/FieldInstance+ opts :- ClassifyOpts](classify! field opts(or (:fingerprint field)(when (qp.store/initialized?)@@ -80,7 +86,7 @@(sync.db/field-fingerprint (u/the-id field)))))([field :- i/FieldInstance- {:keys [exists-name]}+ {:keys [exists-name]} :- ClassifyOptsfingerprint :- [:maybe ::lib.schema.metadata.fingerprint/fingerprint]](sync-util/with-error-handling (format "Error classifying %s" (sync-util/name-for-logging field))(let [classified (analyze/run-classifiers field fingerprint)@@ -135,10 +141,13 @@table(save-model-updates! table updated-table))))+(def ^:private LogProgressFn+ [:=> [:cat :string [:schema i/TableInstance]] :nil])+(mu/defn classify-tables-for-db!"Classify all tables found in a given database"- [database :- i/DatabaseInstance- log-progress-fn]+ [database :- i/DatabaseInstance+ log-progress-fn :- LogProgressFn](let [tables (sync-util/reducible-sync-tables database)](reduce (fn [acc table](let [result (classify-table! table)]@@ -151,8 +160,8 @@(mu/defn classify-fields-for-db!"Classify all fields found in a given database"- [database :- i/DatabaseInstance- log-progress-fn]+ [database :- i/DatabaseInstance+ log-progress-fn :- LogProgressFn](let [tables (sync-util/reducible-sync-tables database)](transduce (map (fn [table](let [result (classify-fields! table)]diff --git a/src/metabase/sync/analyze/data_sensitivity.clj b/src/metabase/sync/analyze/data_sensitivity.cljindex b2a8c7c19e2..05d192874b8 100644--- a/src/metabase/sync/analyze/data_sensitivity.clj+++ b/src/metabase/sync/analyze/data_sensitivity.clj@@ -31,7 +31,7 @@{:fields-scanned 0 :fields-labeled 0 :fields-failed 0})(def ^:private ScanOptions- [:map+ [:map {:closed true}[:force? {:optional true} [:maybe :boolean]][:ignore-setting? {:optional true} [:maybe :boolean]]])@@ -40,8 +40,8 @@(not (sync.settings/data-sensitivity-scan-enabled))))(mu/defn- fields-to-scan :- [:sequential i/FieldInstance]- [table :- i/TableInstance- force?]+ [table :- i/TableInstance+ force? :- [:maybe :boolean]](sync.db/fields-to-scan-for-data-sensitivity (u/the-id table) force?))(mu/defn- classify-and-save!@@ -77,14 +77,17 @@(mu/defn- table-ids-with-unscanned-fields :- [:maybe [:set pos-int?]][database :- i/DatabaseInstance- force?]+ force? :- [:maybe :boolean]](sync.db/table-ids-with-fields-to-scan-for-data-sensitivity (u/the-id database) force?))+(def ^:private LogProgressFn+ [:=> [:cat :string [:schema i/TableInstance]] :nil])+(mu/defn scan-fields-for-db! :- Stats"Label every unscanned Field in every active table of `database`. `log-fn` is accepted for parity with the otheranalyze steps and not called: the step reports per table through the log, not the progress bar."[database :- i/DatabaseInstance- _log-fn+ _log-fn :- [:maybe LogProgressFn]& {:keys [force? ignore-setting?]} :- [:maybe ScanOptions]](if (skip? ignore-setting?)zero-stats@@ -112,7 +115,7 @@"Scan `database-or-table` for data sensitivity regardless of the setting; `:force?` rescans `:PUBLIC` Fields and`:reset?` clears the classifier's labels first."[database-or-table :- [:or i/DatabaseInstance i/TableInstance]- & {:keys [force? reset?]} :- [:maybe [:map+ & {:keys [force? reset?]} :- [:maybe [:map {:closed true}[:force? {:optional true} [:maybe :boolean]][:reset? {:optional true} [:maybe :boolean]]]]](let [reset (when reset? (reset-data-sensitivity! database-or-table))diff --git a/src/metabase/sync/analyze/fingerprint.clj b/src/metabase/sync/analyze/fingerprint.cljindex 27dc192d267..c592c1b6a09 100644--- a/src/metabase/sync/analyze/fingerprint.clj+++ b/src/metabase/sync/analyze/fingerprint.clj@@ -46,7 +46,7 @@(sync.db/set-fields-fingerprint-version! ids i/*latest-fingerprint-version*)))(mr/def ::FingerprintStats- [:map+ [:map {:closed true}[:no-data-fingerprints ms/IntGreaterThanOrEqualToZero][:failed-fingerprints ms/IntGreaterThanOrEqualToZero][:updated-fingerprints ms/IntGreaterThanOrEqualToZero]@@ -158,7 +158,7 @@(empty-stats-map 0)))))(def ^:private LogProgressFn- [:=> [:cat :string [:schema i/TableInstance]] :any])+ [:=> [:cat :string [:schema i/TableInstance]] :nil])(mu/defn- fingerprint-fields-for-db!*"Invokes `fingerprint-table!` on every table in `database`"@@ -168,7 +168,7 @@([database :- i/DatabaseInstancelog-progress-fn :- LogProgressFn- continue? :- [:=> [:cat ::FingerprintStats] :any]]+ continue? :- [:=> [:cat ::FingerprintStats] :boolean]](let [tables (if *refingerprint?*(sync-util/refingerprint-reducible-sync-tables database)(sync-util/reducible-sync-tables database))]diff --git a/src/metabase/sync/analyze/interestingness.clj b/src/metabase/sync/analyze/interestingness.cljindex f7fb88bea4e..7f6a098a8e2 100644--- a/src/metabase/sync/analyze/interestingness.clj+++ b/src/metabase/sync/analyze/interestingness.clj@@ -76,10 +76,13 @@{:fields-scored 0 :fields-failed 0}(sync.db/unscored-fields-for-database-reducible (u/the-id database))))+(def ^:private LogProgressFn+ [:=> [:cat :string [:schema i/TableInstance]] :nil])+(mu/defn score-fields-for-db!"Score interestingness for all qualifying Fields in `database`."[database :- i/DatabaseInstance- log-progress-fn]+ log-progress-fn :- LogProgressFn](let [tables (sync-util/reducible-sync-tables database)per-table-stats (transduce (map (fn [table](let [result (score-fields! table)]diff --git a/src/metabase/sync/db.clj b/src/metabase/sync/db.cljindex d2e6fb1bccc..716cf466e4b 100644--- a/src/metabase/sync/db.clj+++ b/src/metabase/sync/db.clj@@ -75,7 +75,7 @@(mu/defn table"The Table with `table-id`, or nil."- [table-id :- ::lib.schema.id/table]+ [table-id :- [:maybe ::lib.schema.id/table]](t2/select-one :model/Table :id table-id {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))(mu/defn table-in-database@@ -185,7 +185,7 @@"Reducible synced Tables of the Database with `database-id` ordered by schema and name, optionally narrowed to`schema-names` and/or `table-names`."[database-id :- ::lib.schema.id/database- schema-names :- [:maybe [:or [:set :string] [:sequential :string]]]+ schema-names :- [:maybe [:or [:set [:maybe :string]] [:sequential [:maybe :string]]]]table-names :- [:maybe [:or [:set :string] [:sequential :string]]]](t2/reducible-select :model/Table:db_id database-iddiff --git a/src/metabase/sync/fetch_metadata.clj b/src/metabase/sync/fetch_metadata.cljindex afa7c6e7194..4f91267c6b8 100644--- a/src/metabase/sync/fetch_metadata.clj+++ b/src/metabase/sync/fetch_metadata.clj@@ -79,7 +79,7 @@"Effectively a wrapper for [[metabase.driver/describe-fields]] that also validates the output against the schema.If the driver doesn't support [[metabase.driver/describe-fields]] it uses [[driver/describe-table]] instead.This will be deprecated in "- [database :- i/DatabaseInstance & {:as args}]+ [database :- i/DatabaseInstance & {:as args} :- [:maybe ::driver/describe-fks.options]](log-if-error "fields-metadata"(let [driver (driver.u/database->driver database)describe-fields-fn (if (driver.u/supports? driver :describe-fields database)@@ -97,7 +97,7 @@(mu/defn fk-metadata"Effectively a wrapper for [[metabase.driver/describe-fks]] that also validates the output against the schema."[database :- i/DatabaseInstance- & {:as args} :- ::driver/describe-fks.options]+ & {:as args} :- [:maybe ::driver/describe-fks.options]](log-if-error "fk-metadata"(let [driver (driver.u/database->driver database)](when (driver.u/supports? driver :metadata/key-constraints database)diff --git a/src/metabase/sync/interface.clj b/src/metabase/sync/interface.cljindex 9e57d301fac..b42dabaeda0 100644--- a/src/metabase/sync/interface.clj+++ b/src/metabase/sync/interface.clj@@ -26,17 +26,18 @@[:ref ::DatabaseMetadataTable])(mr/def ::DatabaseMetadata- [:map+ [:map {:closed true}[:tables [:fn {:error/message "a set, sequential, or reducible collection of tables"}(fn [x] (or (set? x) (sequential? x) (instance? clojure.lang.IReduceInit x)))]]- [:version {:optional true} [:maybe ::lib.schema.common/non-blank-string]]])+ [:version {:optional true} [:maybe ::lib.schema.common/non-blank-string]]+ [:metabase-metadata-tables {:optional true} (ms/InstanceOfClass clojure.lang.Volatile)]])(def DatabaseMetadata"Schema for the expected output of `describe-database`."[:ref ::DatabaseMetadata])(mr/def ::TableMetadataField- [:map+ [:map {:closed true}[:name ::lib.schema.common/non-blank-string][:database-type [:maybe ::lib.schema.common/non-blank-string]] ; blank if the Field is all NULL & untyped, i.e. in Mongo[:base-type ::lib.schema.common/base-type]@@ -51,7 +52,7 @@[:json-unfolding {:optional true} :boolean];; TODO (Cam 8/11/25) -- this should be required to be a sequence of strings but we'll need to go fix some code[:nfc-path {:optional true} [:maybe [:sequential [:or :keyword :string]]]]- [:custom {:optional true} :map]+ [:custom {:optional true} ms/OpaqueJSONObject][:database-default {:optional true} :string][:database-is-auto-increment {:optional true} :boolean][:database-is-generated {:optional true} :boolean]@@ -59,7 +60,11 @@;; nullable for databases that don't support field partition[:database-partitioned {:optional true} [:maybe :boolean]][:database-required {:optional true} :boolean]- [:visibility-type {:optional true} [:maybe :keyword]]])+ [:visibility-type {:optional true} [:maybe :keyword]]+ [:jdbc-type {:optional true} [:maybe :int]]+ [:preview-display {:optional true} :boolean]+ [:table-name {:optional true} ::lib.schema.common/non-blank-string]+ [:table-schema {:optional true} [:maybe ::lib.schema.common/non-blank-string]]])(def TableMetadataField"Schema for a given Field as provided in [[metabase.driver/describe-table]]."@@ -116,7 +121,7 @@[:ref ::TableFKMetadataEntry])(mr/def ::FKMetadataEntry- [:map+ [:map {:closed true}[:fk-table-name ::lib.schema.common/non-blank-string][:fk-table-schema [:maybe ::lib.schema.common/non-blank-string]][:fk-column-name ::lib.schema.common/non-blank-string]diff --git a/src/metabase/sync/sync.clj b/src/metabase/sync/sync.cljindex 8b0c1a2238a..73d2470f974 100644--- a/src/metabase/sync/sync.clj+++ b/src/metabase/sync/sync.clj@@ -65,11 +65,11 @@Please note that this function is *not* what is called by the scheduled tasks; those call different stepsindependently. This function is called when a Database is first added."- ([database]+ ([database :- i/DatabaseInstance](sync-database! database nil))([database :- i/DatabaseInstance- {:keys [scan], :or {scan :full}} :- [:maybe [:map+ {:keys [scan], :or {scan :full}} :- [:maybe [:map {:closed true}[:scan {:optional true} [:maybe [:enum :schema :full]]]]]](tracing/with-span :sync "sync.database" {:db/id (:id database)}(sync-util/sync-operation :sync database (format "Sync %s" (sync-util/name-for-logging database))diff --git a/src/metabase/sync/sync_metadata/fields/sync_metadata.clj b/src/metabase/sync/sync_metadata/fields/sync_metadata.cljindex 1fc75e138b5..43d3de9a1c5 100644--- a/src/metabase/sync/sync_metadata/fields/sync_metadata.clj+++ b/src/metabase/sync/sync_metadata/fields/sync_metadata.clj@@ -128,7 +128,7 @@old-base-typenew-base-type)(field-user-settings/unset-user-settings!- metabase-field [:effective_type :coercion_strategy :semantic_type])+ (select-keys metabase-field [:id]) [:effective_type :coercion_strategy :semantic_type]){:base_type new-base-type:effective_type new-base-type:coercion_strategy nil@@ -145,7 +145,7 @@(common/field-metadata-name-for-logging table metabase-field)old-effective-typenew-base-type)- (field-user-settings/unset-user-settings! metabase-field [:effective_type :coercion_strategy])+ (field-user-settings/unset-user-settings! (select-keys metabase-field [:id]) [:effective_type :coercion_strategy]){:effective_type new-base-type})(when new-semantic-type?(log/infof "Semantic type of %s has changed from '%s' to '%s'."diff --git a/src/metabase/sync/sync_metadata/metabase_metadata.clj b/src/metabase/sync/sync_metadata/metabase_metadata.cljindex e050119aa2b..f5146612321 100644--- a/src/metabase/sync/sync_metadata/metabase_metadata.clj+++ b/src/metabase/sync/sync_metadata/metabase_metadata.clj@@ -19,7 +19,7 @@[metabase.util.malli.schema :as ms]))(def ^:private KeypathComponents- [:map+ [:map {:closed true}[:table-name [:maybe ms/NonBlankString]][:field-name [:maybe ms/NonBlankString]][:k :keyword]])@@ -41,7 +41,7 @@"Set a property for a Field or Table in `database`. Returns `true` if a property was successfully set."[database :- i/DatabaseInstance{:keys [table-name field-name k]} :- KeypathComponents- value]+ value :- [:maybe :string]](boolean;; ignore legacy entries that try to set field_type since it's no longer part of Field(when-not (= k :field_type)@@ -72,7 +72,7 @@This functionality is currently only used by the Sample Database. In order to use this functionality, drivers *must*implement optional fn `:table-rows-seq`."- [driver+ [driver :- :keyworddatabase :- i/DatabaseInstancemetabase-metadata-table :- i/DatabaseMetadataTable](doseq [{:keys [keypath value]} (driver/table-rows-seq driver database metabase-metadata-table)]diff --git a/src/metabase/sync/sync_metadata/tables.clj b/src/metabase/sync/sync_metadata/tables.cljindex 8711d948f68..b5ce3088619 100644--- a/src/metabase/sync/sync_metadata/tables.clj+++ b/src/metabase/sync/sync_metadata/tables.clj@@ -20,7 +20,9 @@[metabase.util.log :as log][metabase.util.malli :as mu][metabase.util.malli.schema :as ms]- [metabase.warehouse-schema.humanization :as warehouse-schema.humanization]))+ [metabase.warehouse-schema.humanization :as warehouse-schema.humanization])+ (:import+ (clojure.lang ITransientSet)))(set! *warn-on-reflection* true)@@ -89,6 +91,28 @@[:name ::lib.schema.common/non-blank-string][:schema [:maybe ::lib.schema.common/non-blank-string]]])+(def ^:private TableMetadata+ "The table map `create-table!`/`create-or-reactivate-table!`/`cruft-dependent-cols` accept: a+ `DatabaseMetadataTable`-ish shape (`:schema` optional, unlike [[i/DatabaseMetadataTable]], since callers can build+ one with just a `:name`), plus the extra keys `create-table!` reads off it before insertion."+ [:map {:closed true}+ [:name ::lib.schema.common/non-blank-string]+ [:schema {:optional true} [:maybe ::lib.schema.common/non-blank-string]]+ [:is_writable {:optional true} [:maybe :boolean]]+ [:estimated_row_count {:optional true} [:maybe :int]]+ [:database_require_filter {:optional true} [:maybe :boolean]]+ [:description {:optional true} [:maybe :string]]+ [:visibility_type {:optional true} [:maybe [:or :keyword :string]]]+ [:display_name {:optional true} [:maybe :string]]+ [:initial_sync_status {:optional true} [:maybe :string]]+ [:field_order {:optional true} [:maybe :keyword]]+ [:data_source {:optional true} [:maybe :keyword]]+ [:data_authority {:optional true} [:maybe :keyword]]])++(def ^:private TableMetadataOrInstance+ "Either a [[TableMetadata]] fresh from the driver or a Table row."+ [:or TableMetadata :metabase.warehouse-schema.schema/table])+(mu/defn- update-database-metadata!"If there is a version in the db-metadata update the DB to have that in the DB model"[database :- i/DatabaseInstance@@ -98,7 +122,7 @@{:details (assoc (:details database) :version (:version db-metadata))}))(mu/defn- cruft-dependent-cols :- :map- [{table-name :name :as table} :- :map+ [{table-name :name :as table} :- TableMetadataOrInstancedatabase :- i/DatabaseInstancesync-stage :- [:enum ::reactivate ::create ::update]](let [is-crufty? (if (and (= sync-stage ::update)@@ -129,7 +153,7 @@"Creates a new table in the database, ready to be synced.Throws an exception if there is already a table with the same name, schema and database ID."[database :- i/DatabaseInstance- table :- :map]+ table :- TableMetadata](sync.db/insert-table!(merge (cruft-dependent-cols table database ::create){:active true@@ -167,7 +191,7 @@(mu/defn create-or-reactivate-table!"Create a single new table in the database, or mark it as active if a matching inactive one exists."[database :- i/DatabaseInstance- {schema :schema table-name :name :as table} :- :map]+ {schema :schema table-name :name :as table} :- TableMetadata](if-let [existing (sync.db/inactive-table-by-schema-and-name (u/the-id database) schema table-name)](reactivate-table! database existing)(create-table! database table)))@@ -193,7 +217,7 @@(mu/defn- table-name-or-schema-too-long? :- :boolean"Whether `table`'s name or schema is too long to store in the application DB (see `table-name-max-length` /`table-schema-max-length`)."- [{table-name :name, table-schema :schema} :- :map]+ [{table-name :name, table-schema :schema} :- i/DatabaseMetadataTable](boolean(or (< table-name-max-length (count table-name))(< table-schema-max-length (count (or table-schema ""))))))@@ -301,14 +325,14 @@(mu/defn- ignore-table? :- :boolean"Tables we never create `:model/Table` rows for: the special `_metabase_metadata` table (itscontents are applied to other Tables/Fields instead) and temporary transform output tables."- [table :- :map]+ [table :- i/DatabaseMetadataTable](boolean(or (metabase-metadata/is-metabase-metadata-table? table)(sync-util/is-temp-transform-table? table))))(mu/defn- table-name+schema :- TableNameAndSchema"The `{:name :schema}` identity a Table is keyed on during sync."- [table :- :map]+ [table :- TableMetadataOrInstance](select-keys table [:name :schema]))(mu/defn- existing-tables-by-name+schema :- :map@@ -316,7 +340,7 @@indexed by `{:name :schema}`. Queries by name only -- not schema -- since `:schema` can be `nil`and `IN (NULL, ...)` wouldn't match; the exact `(name, schema)` match falls out of the index key."[database :- i/DatabaseInstance- tables]+ tables :- [:set TableMetadataOrInstance]](let [names (into [] (comp (map :name) (distinct)) tables)](if (seq names)(m/index-by table-name+schema@@ -329,7 +353,7 @@"Returns a map of `{old-schema new-schema}` for the app-DB schemas the driver re-qualifies. Streamsthe distinct schemas straight from the DB (selecting only `:schema`) so we never materialize everytable just to read their schemas."- [driver+ [driver :- :keyworddatabase :- i/DatabaseInstance](transduce(comp (map :schema) (distinct))@@ -397,7 +421,7 @@[:created :int] ; running count of created/reactivated tables[:updated :int] ; running count of metadata-updated tables[:complete? :boolean] ; false once any batch fails -- then we don't retire- [:seen :any] ; transient set of reconciled `:model/Table` ids+ [:seen (ms/InstanceOfClass ITransientSet)] ; transient set of reconciled `:model/Table` ids[:metabase-metadata [:vector i/DatabaseMetadataTable]]]) ; captured `_metabase_metadata` table(mu/defn- sync-table-batch! :- SyncContextdiff --git a/src/metabase/sync/task/sync_databases.clj b/src/metabase/sync/task/sync_databases.cljindex 86b95ec252b..000b707cdd9 100644--- a/src/metabase/sync/task/sync_databases.clj+++ b/src/metabase/sync/task/sync_databases.clj@@ -40,7 +40,8 @@(mu/defn- job-context->database-id :- [:maybe ::lib.schema.id/database]"Get the Database ID referred to in `job-context`."- [job-context]+ [job-context :- [:fn {:error/message "satisfies clojurewerkz.quartzite.conversion/JobDataMapConversion"}+ #(satisfies? qc/JobDataMapConversion %)]](u/the-id (get (qc/from-job-data job-context) "db-id")));; The DisallowConcurrentExecution on the two defrecords below attaches an annotation to the generated class that willdiff --git a/src/metabase/sync/task/sync_databases_trigger.clj b/src/metabase/sync/task/sync_databases_trigger.cljindex 369da9fbe2d..da2b592a014 100644--- a/src/metabase/sync/task/sync_databases_trigger.clj+++ b/src/metabase/sync/task/sync_databases_trigger.clj@@ -29,7 +29,7 @@(def TaskInfo"One-off schema for information about the various sync tasks we run for a DB."- [:map+ [:map {:closed true}[:key :keyword][:db-schedule-column :keyword][:name :string]])diff --git a/src/metabase/sync/util.clj b/src/metabase/sync/util.cljindex 3e819130ac8..ec136121068 100644--- a/src/metabase/sync/util.clj+++ b/src/metabase/sync/util.clj@@ -67,6 +67,9 @@{:error/message "Sync event deriving from :metabase.sync.util/event"}#(events/isa? % ::event)]])+(def ^:private DatabaseOrId+ [:or ::lib.schema.id/database :metabase.warehouses.schema/database-or-metadata])+;;; +----------------------------------------------------------------------------------------------------------------+;;; | SYNC OPERATION "MIDDLEWARE" |;;; +----------------------------------------------------------------------------------------------------------------+@@ -127,7 +130,9 @@`database-id`. `f` is executed between the logging of the two events."{:style/indent [:form]};; we can do everyone a favor and infer the name of the individual begin and sync events- ([event-name-prefix database-or-id f]+ ([event-name-prefix :- :keyword+ database-or-id :- DatabaseOrId+ f :- fn?](letfn [(event-keyword [prefix suffix](keyword (or (namespace event-name-prefix) "event")(str (name prefix) suffix)))]@@ -139,8 +144,8 @@([begin-event-name :- Topicend-event-name :- Topic- database-or-id- f]+ database-or-id :- DatabaseOrId+ f :- fn?](fn [](let [start-time (System/nanoTime)tracking-hash (str (random-uuid))]@@ -461,11 +466,8 @@"Returns a reducible of all the Tables that should go through the sync processes for `database-or-id`.Returns tables ordered by `[schema name]` so results match the order expected by [[metabase.driver/describe-fks]]."- [database-or-id :- [:or- ::lib.schema.id/database- [:map- [:id ::lib.schema.id/database]]]- & {:keys [schema-names table-names]} :- ::driver/describe-fks.options]+ [database-or-id :- DatabaseOrId+ & {:keys [schema-names table-names]} :- [:maybe ::driver/describe-fks.options]](eduction (map t2.realize/realize)(sync.db/sync-tables-reducible (u/the-id database-or-id) schema-names table-names)))@@ -529,13 +531,52 @@(def ^:private TimedSyncMetadata"Metadata common to both sync steps and an entire sync/analyze operation run"- [:map+ [:map {:closed true}[:start-time (ms/InstanceOfClass Temporal)][:end-time {:optional true} (ms/InstanceOfClass Temporal)]])+(def ^:private StepStats+ "Step-specific stats a `sync-fn` may add to its `StepRunMetadata`, across all of the sync/analyze steps."+ [:map {:closed true}+ [:added-indexes {:optional true} :int]+ [:cloud {:optional true} :boolean]+ [:created {:optional true} :int]+ [:deleted {:optional true} :int]+ [:errors {:optional true} :int]+ [:failed-fingerprints {:optional true} :int]+ [:fields-classified {:optional true} :int]+ [:fields-failed {:optional true} :int]+ [:fields-labeled {:optional true} :int]+ [:fields-scanned {:optional true} :int]+ [:fields-scored {:optional true} :int]+ [:fingerprints-attempted {:optional true} :int]+ [:flavor {:optional true} :string]+ [:no-data-fingerprints {:optional true} :int]+ [:probed {:optional true} :int]+ [:queries {:optional true} :int]+ [:removed-indexes {:optional true} :int]+ [:semantic-version {:optional true} [:or+ [:sequential :int]+ [:map {:closed true} [:major :int] [:minor :int]]]]+ [:tables-classified {:optional true} :int]+ [:throwable {:optional true} [:maybe (ms/InstanceOfClass Throwable)]]+ [:timezone-id {:optional true} [:maybe :string]]+ [:total-failed {:optional true} :int]+ [:total-fields {:optional true} :int]+ [:total-fks {:optional true} :int]+ [:total-indexes {:optional true} :int]+ [:total-tables {:optional true} :int]+ [:updated {:optional true} :int]+ [:updated-fields {:optional true} :int]+ [:updated-fingerprints {:optional true} :int]+ [:updated-fks {:optional true} :int]+ [:updated-tables {:optional true} :int]+ [:version {:optional true} :string]])+(mr/def ::StepRunMetadata[:mergeTimedSyncMetadata+ StepStats[:map[:log-summary-fn [:maybe [:=> [:cat [:ref ::StepRunMetadata]] :string]]]]])@@ -570,7 +611,7 @@"Defines a step. `:sync-fn` runs the step, returns a map that contains step specific metadata. `log-summary-fn`takes that metadata and turns it into a string for logging. `:essential?` marks a step whose failure leaves thedatabase unusable (e.g. field sync), so initial sync should be reported as failed rather than complete."- [:map+ [:map {:closed true}[:sync-fn [:=> [:cat StepRunMetadata] i/DatabaseInstance]][:step-name :string][:log-summary-fn [:maybe LogSummaryFunction]]diff --git a/src/metabase/task_history/models/task_history.clj b/src/metabase/task_history/models/task_history.cljindex a2098f04fc6..032faeabf57 100644--- a/src/metabase/task_history/models/task_history.clj+++ b/src/metabase/task_history/models/task_history.clj@@ -11,6 +11,7 @@[metabase.premium-features.core :as premium-features][metabase.task-history.db :as task-history.db][metabase.task-history.models.task-run :as task-run]+ [metabase.task-history.schema :as task-history.schema][metabase.util :as u][metabase.util.json :as json][metabase.util.malli :as mu]@@ -124,9 +125,11 @@[:map {:closed true}[:task ms/NonBlankString] ; task name, i.e. `send-pulses`. Conventionally lisp-cased[:db_id {:optional true} [:maybe :int]] ; DB involved, for sync operations or other tasks where this is applicable.- [:on-success-info {:optional true} [:maybe [:=> [:cat TaskHistoryCallBackInfo :any] :map]]]- [:on-fail-info {:optional true} [:maybe [:=> [:cat TaskHistoryCallBackInfo :any] :map]]]- [:task_details {:optional true} [:maybe :map]]]) ; additional map of details to include in the recorded row+ [:on-success-info {:optional true} [:maybe [:=> [:cat TaskHistoryCallBackInfo :any]+ ::task-history.schema/task-history.update]]]+ [:on-fail-info {:optional true} [:maybe [:=> [:cat TaskHistoryCallBackInfo (ms/InstanceOfClass Throwable)]+ ::task-history.schema/task-history.update]]]+ [:task_details {:optional true} [:maybe ::task-history.schema/task-history.task-details]]]) ; additional map of details to include in the recorded row(defn- ns->ms [nanoseconds](long (/ nanoseconds 1e6)))@@ -207,7 +210,8 @@(mu/defn do-with-task-history"Impl for `with-task-history` macro; see documentation below."- [info :- TaskHistoryInfo f]+ [info :- TaskHistoryInfo+ f :- ifn?](let [on-success-info (or (:on-success-info info) (fn [& args] (first args)))on-fail-info (or (:on-fail-info info) (fn [& args] (first args)))info (dissoc info :on-success-info :on-fail-info)diff --git a/src/metabase/task_history/schema.clj b/src/metabase/task_history/schema.cljindex 6c3047a5948..bf00459c416 100644--- a/src/metabase/task_history/schema.clj+++ b/src/metabase/task_history/schema.clj@@ -5,27 +5,172 @@[metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms]))+(mr/def ::retry-error+ "One entry of a `:retry_errors` list: a retry attempt's error message and when it happened."+ [:map {:closed true}+ [:message :string]+ [:timestamp ms/TemporalInstant]])++(mr/def ::task-details.channel-send+ [:map {:closed true}+ [:retry_config {:optional true} [:map {:closed true}+ [:max-retries :int]+ [:initial-interval-millis :int]+ [:multiplier number?]+ [:jitter-factor number?]+ [:max-interval-millis :int]]]+ [:channel_id {:optional true} [:maybe ms/PositiveInt]]+ [:channel_type {:optional true} :keyword]+ [:template_id {:optional true} [:maybe ms/PositiveInt]]+ [:notification_id {:optional true} [:maybe ms/PositiveInt]]+ [:notification_type {:optional true} :keyword]+ [:recipient_ids {:optional true} [:sequential [:maybe ms/PositiveInt]]]+ [:attempted_retries {:optional true} :int]+ [:retry_errors {:optional true} [:sequential ::retry-error]]])++(mr/def ::task-details.notification-send+ [:map {:closed true}+ [:notification_id [:maybe ms/PositiveInt]]+ [:notification_handlers [:sequential [:map {:closed true}+ [:id {:optional true} [:maybe ms/PositiveInt]]+ [:channel_type {:optional true} [:maybe :keyword]]+ [:channel_id {:optional true} [:maybe ms/PositiveInt]]+ [:template_id {:optional true} [:maybe ms/PositiveInt]]]]]])++(mr/def ::task-details.notification-trigger+ [:map {:closed true}+ [:trigger_type [:enum :notification-subscription/cron :notification-subscription/system-event]]+ [:notification_ids [:sequential ms/PositiveInt]]+ [:notification_subscription_id {:optional true} ms/PositiveInt]+ [:cron_schedule {:optional true} [:maybe :string]]+ [:event_name {:optional true} :keyword]])++(mr/def ::task-details.send-pulse+ [:map {:closed true}+ [:pulse-id ms/PositiveInt]+ [:channel-ids [:maybe [:sequential ms/PositiveInt]]]])++(mr/def ::task-details.run-transforms+ [:map {:closed true}+ [:job-id ms/PositiveInt]+ [:run-method [:= :cron]]+ [:skipped-reason {:optional true} :string]])++(mr/def ::task-details.remote-sync-auto-import+ [:map {:closed true}+ [:task-id ms/PositiveInt]])++(mr/def ::task-details.persist-refresh+ [:map {:closed true}+ [:success :int]+ [:error :int]+ [:skipped {:optional true} :int]+ [:trigger {:optional true} [:maybe (ms/InstanceOfClass org.quartz.Trigger)]]+ [:error-details {:optional true} [:sequential [:map {:closed true}+ [:persisted-info-id ms/PositiveInt]+ [:error {:optional true} [:maybe :string]]]]]])++(mr/def ::task-details.sync-step+ "The `:task_details` of a sync/analyze step: the union of the count/diagnostic keys any step+ (`metabase.sync.util/run-step-with-metadata`) can report."+ [:map {:closed true}+ [:version {:optional true} [:maybe :string]]+ [:timezone-id {:optional true} [:maybe :string]]+ [:total-tables {:optional true} :int]+ [:updated-tables {:optional true} :int]+ [:tables-classified {:optional true} :int]+ [:total-fields {:optional true} :int]+ [:updated-fields {:optional true} :int]+ [:fields-classified {:optional true} :int]+ [:fields-scored {:optional true} :int]+ [:fields-scanned {:optional true} :int]+ [:fields-labeled {:optional true} :int]+ [:fields-failed {:optional true} :int]+ [:total-fks {:optional true} :int]+ [:updated-fks {:optional true} :int]+ [:total-failed {:optional true} :int]+ [:total-indexes {:optional true} :int]+ [:added-indexes {:optional true} :int]+ [:removed-indexes {:optional true} :int]+ [:fingerprints-attempted {:optional true} :int]+ [:updated-fingerprints {:optional true} :int]+ [:no-data-fingerprints {:optional true} :int]+ [:failed-fingerprints {:optional true} :int]+ [:created {:optional true} :int]+ [:updated {:optional true} :int]+ [:deleted {:optional true} :int]+ [:errors {:optional true} :int]+ [:probed {:optional true} :int]+ [:queries {:optional true} :int]+ [:flavor {:optional true} [:maybe :string]]+ [:semantic-version {:optional true} [:or+ [:sequential :int]+ [:map {:closed true} [:major :int] [:minor :int]]]]+ [:cloud {:optional true} :boolean]])++(mr/def ::task-details.empty+ [:map {:closed true}])++(mr/def ::task-details.failure+ "The `:task_details` `do-with-task-history` records when the task throws: the caller's own `:task_details` (any+ shape above) nested under `:original-info`, plus the exception."+ [:map {:closed true}+ [:status [:enum :failed "failed"]]+ [:exception {:optional true} [:or :string (ms/InstanceOfClass Class)]]+ [:message {:optional true} [:maybe :string]]+ [:stacktrace {:optional true} [:maybe [:sequential :string]]]+ [:ex-data {:optional true} [:maybe ms/ExceptionData]]+ [:original-info {:optional true} [:maybe [:ref ::task-history.task-details]]]+ [:reason {:optional true} :string]+ [:attempted_retries {:optional true} :int]+ [:retry_errors {:optional true} [:sequential ::retry-error]]])+(mr/def ::task-history.task-details- "The `:task_details` column of a TaskHistory, decoded."- :map)+ "The `:task_details` column of a TaskHistory, decoded: the union of the shapes recorded for each task name (see+ the `with-task-history` call sites)."+ [:or+ ::task-details.channel-send+ ::task-details.notification-send+ ::task-details.notification-trigger+ ::task-details.send-pulse+ ::task-details.run-transforms+ ::task-details.remote-sync-auto-import+ ::task-details.persist-refresh+ ::task-details.sync-step+ ::task-details.failure+ ::task-details.empty])++(mr/def ::task-history.log.trunc+ "The `:trunc` entry of a [[task-history.log]]: bookkeeping for messages dropped once the in-memory log queue fills+ up."+ [:map {:closed true}+ [:levels [:map {:closed true}+ [:trace {:optional true} :int]+ [:debug {:optional true} :int]+ [:info {:optional true} :int]+ [:warn {:optional true} :int]+ [:error {:optional true} :int]+ [:fatal {:optional true} :int]]]+ [:start-timestamp [:maybe :string]]+ [:last-timestamp [:maybe :string]]])(mr/def ::task-history.log"One entry of the `:logs` column of a TaskHistory, decoded."- :map)+ [:map {:closed true}+ [:level {:optional true} [:enum :trace :debug :info :warn :error :fatal]]+ [:timestamp {:optional true} :string]+ [:fqns {:optional true} :string]+ [:msg {:optional true} :string]+ [:process_uuid {:optional true} :string]+ [:exception {:optional true} [:sequential :string]]+ [:trunc {:optional true} ::task-history.log.trunc]])(mr/def ::task-history"A TaskHistory as selected from the app DB: every column of `:task_history`."- [:map {:closed true}- [:id ms/PositiveInt]- [:task :string]- [:db_id [:maybe ::lib.schema.id/database]]- [:started_at ms/TemporalInstant]- [:ended_at [:maybe ms/TemporalInstant]]- [:duration [:maybe :int]]- [:task_details [:maybe ::task-history.task-details]]- [:status [:or :keyword :string]]- [:run_id [:maybe ms/PositiveInt]]- [:logs [:maybe [:sequential ::task-history.log]]]])+ [:merge+ ::task-history.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::task-history.update"What an update (or insert) of a TaskHistory accepts: every column of `:task_history` except `id`, all optional."@@ -42,17 +187,10 @@(mr/def ::task-run"A TaskRun as selected from the app DB: every column of `:task_run`."- [:map {:closed true}- [:id ms/PositiveInt]- [:run_type [:or :keyword :string]]- [:entity_type [:or :keyword :string]]- [:entity_id ms/PositiveInt]- [:started_at ms/TemporalInstant]- [:ended_at [:maybe ms/TemporalInstant]]- [:status [:or :keyword :string]]- [:process_uuid :string]- [:updated_at ms/TemporalInstant]- [:notification_id [:maybe ms/PositiveInt]]])+ [:merge+ ::task-run.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::task-run.update"What an update (or insert) of a TaskRun accepts: every column of `:task_run` except `id`, all optional."diff --git a/src/metabase/tiles/api.clj b/src/metabase/tiles/api.cljindex 397a22705b5..f1beba6f89a 100644--- a/src/metabase/tiles/api.clj+++ b/src/metabase/tiles/api.clj@@ -70,7 +70,9 @@[query :- ::lib.schema/querylat-field :- ::lib.schema.metadata/columnlon-field :- ::lib.schema.metadata/column- x y zoom]+ x :- ms/Int+ y :- ms/Int+ zoom :- ms/Int](let [top-left (x+y+zoom->lat-lon x y zoom)bottom-right (x+y+zoom->lat-lon (inc x) (inc y) zoom)inside-filter (lib/inside@@ -150,10 +152,10 @@- add [:inside lat lon bounding-region coordings] filter- limit query results to `tile-coordinate-limit` number of results- only select lat and lon fields rather than entire query's fields"- [query :- :map- zoom- x- y+ [query :- [:maybe [:or [:= {} {}] ::mbql.s/Query ::lib.schema/query]]+ zoom :- ms/Int+ x :- ms/Int+ y :- ms/Intlat-field-legacy-ref :- ::legacy-reflon-field-legacy-ref :- ::legacy-ref](let [query (-> querydiff --git a/src/metabase/timeline/api/timeline_event.clj b/src/metabase/timeline/api/timeline_event.cljindex c903c4027ed..784222cffa4 100644--- a/src/metabase/timeline/api/timeline_event.clj+++ b/src/metabase/timeline/api/timeline_event.clj@@ -90,7 +90,7 @@timeline-event-updates (cond-> timeline-event-updates(boolean timestamp) (update :timestamp u.date/parse))](collection/check-allowed-to-change-collection existing timeline-event-updates)- (when (api/column-will-change? :timeline_id existing timeline-event-updates)+ (when (api/column-will-change? (:timeline_id existing) (get timeline-event-updates :timeline_id ::api/not-provided))(api/write-check :model/Timeline (:timeline_id timeline-event-updates)));; todo: if we accept a new timestamp, must we require a timezone? gut says yes?(timeline.db/update-timeline-event! iddiff --git a/src/metabase/timeline/models/timeline.clj b/src/metabase/timeline/models/timeline.cljindex 7e24bd48b7b..e97ccf8471c 100644--- a/src/metabase/timeline/models/timeline.clj+++ b/src/metabase/timeline/models/timeline.clj@@ -5,6 +5,7 @@[metabase.models.serialization :as serdes][metabase.timeline.db :as timeline.db][metabase.timeline.models.timeline-event :as timeline-event]+ [metabase.timeline.schema][methodical.core :as methodical][toucan2.core :as t2]))diff --git a/src/metabase/timeline/models/timeline_event.clj b/src/metabase/timeline/models/timeline_event.cljindex c9c5131193a..01e70c45390 100644--- a/src/metabase/timeline/models/timeline_event.clj+++ b/src/metabase/timeline/models/timeline_event.clj@@ -3,6 +3,7 @@[metabase.models.interface :as mi][metabase.models.serialization :as serdes][metabase.timeline.db :as timeline.db]+ [metabase.timeline.schema][methodical.core :as methodical][toucan2.core :as t2]))diff --git a/src/metabase/timeline/schema.clj b/src/metabase/timeline/schema.cljindex f8f1165bf15..cbfa1881e97 100644--- a/src/metabase/timeline/schema.clj+++ b/src/metabase/timeline/schema.clj@@ -7,18 +7,10 @@(mr/def ::timeline"A Timeline as selected from the app DB: every column of `:timeline`."- [:map {:closed true}- [:id ms/PositiveInt]- [:name :string]- [:description [:maybe :string]]- [:icon :string]- [:collection_id [:maybe ::lib.schema.id/collection]]- [:archived :boolean]- [:creator_id ::lib.schema.id/user]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]- [:default :boolean]- [:entity_id :string]])+ [:merge+ ::timeline.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::timeline.update"What an update (or insert) of a Timeline accepts: every column of `:timeline` except `id`, all optional."@@ -36,19 +28,10 @@(mr/def ::timeline-event"A TimelineEvent as selected from the app DB: every column of `:timeline_event`."- [:map {:closed true}- [:id ms/PositiveInt]- [:timeline_id ms/PositiveInt]- [:name :string]- [:description [:maybe :string]]- [:timestamp ms/TemporalInstant]- [:time_matters :boolean]- [:timezone :string]- [:icon :string]- [:archived :boolean]- [:creator_id ::lib.schema.id/user]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]])+ [:merge+ ::timeline-event.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::timeline-event.update"What an update (or insert) of a TimelineEvent accepts: every column of `:timeline_event` except `id`, all optional."diff --git a/src/metabase/transforms/db.clj b/src/metabase/transforms/db.cljindex 9428d505435..244709e6107 100644--- a/src/metabase/transforms/db.clj+++ b/src/metabase/transforms/db.clj@@ -402,9 +402,11 @@:where [:= :rn [:inline 1]]}))(mu/defn active-run-for-transform- "The active TransformRun of the Transform with `transform-id`, or nil."- [transform-id :- ::lib.schema.id/transform]- (t2/select-one :model/TransformRun :transform_id transform-id :is_active true))+ "The active TransformRun of the Transform with `transform-id`, or nil. `transform-id` is nil for a+ transform that hasn't been saved yet, in which case there is no run either."+ [transform-id :- [:maybe ::lib.schema.id/transform]]+ (when transform-id+ (t2/select-one :model/TransformRun :transform_id transform-id :is_active true)))(mu/defn active-run-ids-of-parent"The IDs of the active TransformRuns whose `parent-column` is `parent-run-id`."@@ -777,7 +779,7 @@(mu/defn table"The Table with `table-id`, or nil."- [table-id :- ::lib.schema.id/table]+ [table-id :- [:maybe ::lib.schema.id/table]](t2/select-one :model/Table :id table-id {:from [(warehouse-schema-overlay/table-query)]}))(mu/defn databasesdiff --git a/src/metabase/transforms/models/transform.clj b/src/metabase/transforms/models/transform.cljindex d2e5eff9b3d..1c5197e0f9f 100644--- a/src/metabase/transforms/models/transform.clj+++ b/src/metabase/transforms/models/transform.clj@@ -18,6 +18,7 @@[metabase.transforms-base.util :as transforms-base.u][metabase.transforms.db :as transforms.db][metabase.transforms.models.transform-run :as transform-run]+ [metabase.transforms.schema][metabase.transforms.util :as transforms.u][metabase.util :as u][metabase.util.log :as log]@@ -282,7 +283,7 @@last-runs (m/index-by :transform_id (transform-run/latest-runs transform-ids))](for [{transform-id :id :as transform} transforms](let [{:keys [status checkpoint_hi_value] :as last-run} (get last-runs transform-id)- transform (assoc transform :last_run (dissoc last-run :last_heartbeat))]+ transform (assoc transform :last_run last-run)](if (and (= status :succeeded) checkpoint_hi_value);; ensure consistency of last_checkpoint_value with last_run(if (:last_checkpoint_value transform)@@ -497,11 +498,12 @@(m/update-existing :source-database import-maybe-int-database-fk)(m/update-existing :source-tables(fn [entries]- (->> (cond-> entries (map? entries) transforms-base.u/source-tables-map->vec)- (mapv (fn [entry]- (-> entry- (m/update-existing :table_id import-maybe-int-table-fk)- (m/update-existing :database_id import-maybe-int-database-fk)))))))+ (let [entries (cond-> entries (map? entries) (update-keys name))]+ (->> (cond-> entries (map? entries) transforms-base.u/source-tables-map->vec)+ (mapv (fn [entry]+ (-> entry+ (m/update-existing :table_id import-maybe-int-table-fk)+ (m/update-existing :database_id import-maybe-int-database-fk))))))))(update-checkpoint-field import-maybe-int-field-fk))))}:target {:export #(serdes/export-mbql (dissoc % :table_id)):import serdes/import-mbql}diff --git a/src/metabase/transforms/models/transform_job.clj b/src/metabase/transforms/models/transform_job.cljindex 4e66aed712d..e8a83a7c8ec 100644--- a/src/metabase/transforms/models/transform_job.clj+++ b/src/metabase/transforms/models/transform_job.clj@@ -10,6 +10,7 @@[metabase.transforms.models.job-run :as transforms.job-run][metabase.transforms.models.transform :as transform][metabase.transforms.schedule :as transforms.schedule]+ [metabase.transforms.schema][metabase.util :as u][metabase.util.i18n :as i18n][methodical.core :as methodical]diff --git a/src/metabase/transforms/models/transform_job_transform_tag.clj b/src/metabase/transforms/models/transform_job_transform_tag.cljindex 6246e1fd00d..54b8868c20d 100644--- a/src/metabase/transforms/models/transform_job_transform_tag.clj+++ b/src/metabase/transforms/models/transform_job_transform_tag.clj@@ -1,6 +1,7 @@(ns metabase.transforms.models.transform-job-transform-tag(:require[metabase.models.serialization :as serdes]+ [metabase.transforms.schema][methodical.core :as methodical][toucan2.core :as t2]))diff --git a/src/metabase/transforms/models/transform_tag.clj b/src/metabase/transforms/models/transform_tag.cljindex 4a58501c706..4764e11a05c 100644--- a/src/metabase/transforms/models/transform_tag.clj+++ b/src/metabase/transforms/models/transform_tag.clj@@ -6,6 +6,7 @@[metabase.models.serialization :as serdes][metabase.transforms.db :as transforms.db][metabase.transforms.models.transform :as transform]+ [metabase.transforms.schema][metabase.util.i18n :as i18n][methodical.core :as methodical][toucan2.core :as t2]))diff --git a/src/metabase/transforms/models/transform_transform_tag.clj b/src/metabase/transforms/models/transform_transform_tag.cljindex 6e4d1a9a1d9..47e04dcfaf5 100644--- a/src/metabase/transforms/models/transform_transform_tag.clj+++ b/src/metabase/transforms/models/transform_transform_tag.clj@@ -1,6 +1,7 @@(ns metabase.transforms.models.transform-transform-tag(:require[metabase.models.serialization :as serdes]+ [metabase.transforms.schema][methodical.core :as methodical][toucan2.core :as t2]))diff --git a/src/metabase/transforms/schema.clj b/src/metabase/transforms/schema.cljindex 2051ccb79c9..da22d706c1c 100644--- a/src/metabase/transforms/schema.clj+++ b/src/metabase/transforms/schema.clj@@ -28,13 +28,23 @@:dispatch :type}["checkpoint" ::checkpoint-strategy]])+(mr/def ::orphaned-query+ "The MBQL 5 query of a transform whose source database was deleted, kept as a breadcrumb with its `:database` nulled."+ [:map {:closed true}+ [:lib/type [:= {:decode/normalize lib.schema.common/normalize-keyword} :mbql/query]]+ [:database :nil]+ [:stages [:ref :metabase.lib.schema/stages]]+ [:lib/metadata {:optional true} [:ref :metabase.lib.schema.metadata/metadata-provider]]])+(mr/def ::transform-source[:multi {:decode/normalize lib.schema.common/normalize-map-no-kebab-case:dispatch (comp keyword :type)}[:query[:map {:closed true}[:type {:decode/normalize lib.schema.common/normalize-keyword} [:= :query]]- [:query ::lib-be.schema/maybe-legacy-query]+ [:query [:or+ ::orphaned-query+ ::lib-be.schema/maybe-legacy-query]][:source-incremental-strategy {:optional true} ::source-incremental-strategy]]][:python[:map {:closed true}@@ -68,14 +78,14 @@(mr/def ::table-target[:map {:closed true}- [:database {:optional true} :int]+ [:database {:optional true} [:maybe :int]][:type [:= "table"]][:schema {:optional true} [:maybe ms/NonBlankString]][:name :string]])(mr/def ::table-incremental-target[:map {:closed true}- [:database {:optional true} :int]+ [:database {:optional true} [:maybe :int]][:type [:= "table-incremental"]][:schema {:optional true} [:maybe ms/NonBlankString]][:name :string]@@ -91,37 +101,40 @@(mr/def ::transform.source"The `:source` column of a Transform, decoded."- :map)+ ::transform-source)(mr/def ::transform.target"The `:target` column of a Transform, decoded."- :map)+ ::transform-target)(mr/def ::transform.table-dependency"One entry of the `:table_dependencies` column of a Transform, decoded."- :map)+ [:or+ [:map {:closed true} [:table ::lib.schema.id/table]]+ [:map {:closed true} [:transform ::lib.schema.id/transform]]])++(mr/def ::transform.owner+ "The `:owner` hydrated onto a Transform: its owning User, or just the `:email` of an owner outside Metabase."+ [:or+ :metabase.users.schema/user+ [:map {:closed true} [:email :string]]])(mr/def ::transform- "A Transform as selected from the app DB: every column of `:transform`."- [:map {:closed true}- [:id ::lib.schema.id/transform]- [:name :string]- [:description [:maybe :string]]- [:source ::transform.source]- [:target ::transform.target]- [:entity_id :string]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]- [:source_type [:or :keyword :string]]- [:creator_id ::lib.schema.id/user]- [:source_database_id [:maybe ::lib.schema.id/database]]- [:collection_id [:maybe ::lib.schema.id/collection]]- [:owner_user_id [:maybe ::lib.schema.id/user]]- [:owner_email [:maybe :string]]- [:target_db_id [:maybe ::lib.schema.id/database]]- [:last_checkpoint_value [:maybe :string]]- [:target_table_id [:maybe ::lib.schema.id/table]]- [:table_dependencies [:maybe [:sequential ::transform.table-dependency]]]])+ "A Transform as selected from the app DB: every column of `:transform`, plus `:creator`, `:table`, `:last_run`,+ `:collection`, and `:owner` some callers hydrate onto it."+ [:merge+ ::transform.update+ [:map {:closed true}+ [:id ::lib.schema.id/transform]+ [:creator {:optional true} [:maybe :metabase.users.schema/user]]+ [:table {:optional true} [:maybe [:ref :metabase.warehouse-schema.schema/table]]]+ [:last_run {:optional true} [:maybe ::transform-run]]+ [:collection {:optional true} [:maybe :metabase.collections.schema/collection-or-root]]+ [:owner {:optional true} [:maybe ::transform.owner]]+ [:can_read {:optional true} :boolean]+ [:can_write {:optional true} :boolean]+ [:can_execute {:optional true} :boolean]+ [:tag_ids {:optional true} [:maybe [:sequential ms/PositiveInt]]]]])(mr/def ::transform.update"What an update (or insert) of a Transform accepts: every column of `:transform` except `id`, all optional, plus `:run_trigger` consumed by the model's hooks."@@ -147,22 +160,10 @@(mr/def ::transform-dag-run"A TransformDagRun as selected from the app DB: every column of `:transform_dag_run`."- [:map {:closed true}- [:id ms/PositiveInt]- [:source_transform_id [:maybe ::lib.schema.id/transform]]- [:source_transform_name [:maybe :string]]- [:source_transform_entity_id [:maybe :string]]- [:direction [:or :keyword :string]]- [:transform_count [:maybe :int]]- [:status [:or :keyword :string]]- [:is_active [:maybe :boolean]]- [:start_time ms/TemporalInstant]- [:end_time [:maybe ms/TemporalInstant]]- [:message [:maybe :string]]- [:user_id [:maybe ::lib.schema.id/user]]- [:last_heartbeat ms/TemporalInstant]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]])+ [:merge+ ::transform-dag-run.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::transform-dag-run.update"What an update (or insert) of a TransformDagRun accepts: every column of `:transform_dag_run` except `id`, all optional."@@ -184,17 +185,10 @@(mr/def ::transform-job"A TransformJob as selected from the app DB: every column of `:transform_job`."- [:map {:closed true}- [:id ms/PositiveInt]- [:name [:or :string mu/localized-string-schema]]- [:description [:maybe [:or :string mu/localized-string-schema]]]- [:schedule :string]- [:entity_id :string]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]- [:built_in_type [:maybe [:or :keyword :string]]]- [:ui_display_type [:or :keyword :string]]- [:active :boolean]])+ [:merge+ ::transform-job.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::transform-job.update"What an update (or insert) of a TransformJob accepts: every column of `:transform_job` except `id`, all optional."@@ -211,20 +205,10 @@(mr/def ::transform-job-run"A TransformJobRun as selected from the app DB: every column of `:transform_job_run`."- [:map {:closed true}- [:id ms/PositiveInt]- [:job_id ms/PositiveInt]- [:run_method [:or :keyword :string]]- [:status [:or :keyword :string]]- [:is_active [:maybe :boolean]]- [:start_time ms/TemporalInstant]- [:end_time [:maybe ms/TemporalInstant]]- [:message [:maybe :string]]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]- [:last_heartbeat ms/TemporalInstant]- [:job_name [:maybe :string]]- [:job_entity_id [:maybe :string]]])+ [:merge+ ::transform-job-run.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::transform-job-run.update"What an update (or insert) of a TransformJobRun accepts: every column of `:transform_job_run` except `id`, all optional."@@ -244,12 +228,10 @@(mr/def ::transform-job-transform-tag"A TransformJobTransformTag as selected from the app DB: every column of `:transform_job_transform_tag`."- [:map {:closed true}- [:id ms/PositiveInt]- [:job_id ms/PositiveInt]- [:tag_id ms/PositiveInt]- [:entity_id :string]- [:position :int]])+ [:merge+ ::transform-job-transform-tag.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::transform-job-transform-tag.update"What an update (or insert) of a TransformJobTransformTag accepts: every column of `:transform_job_transform_tag` except `id`, all optional."@@ -261,25 +243,10 @@(mr/def ::transform-run"A TransformRun as selected from the app DB: every column of `:transform_run`."- [:map {:closed true}- [:id ms/PositiveInt]- [:transform_id [:maybe ::lib.schema.id/transform]]- [:run_method [:or :keyword :string]]- [:status [:or :keyword :string]]- [:is_active [:maybe :boolean]]- [:start_time ms/TemporalInstant]- [:end_time [:maybe ms/TemporalInstant]]- [:message [:maybe :string]]- [:user_id [:maybe ::lib.schema.id/user]]- [:transform_name [:maybe :string]]- [:transform_entity_id [:maybe :string]]- [:checkpoint_filter_field_id [:maybe ::lib.schema.id/field]]- [:checkpoint_lo_value [:maybe :string]]- [:checkpoint_hi_value [:maybe :string]]- [:metered_as [:maybe :string]]- [:last_heartbeat ms/TemporalInstant]- [:job_run_id [:maybe ms/PositiveInt]]- [:dag_run_id [:maybe ms/PositiveInt]]])+ [:merge+ ::transform-run.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::transform-run.update"What an update (or insert) of a TransformRun accepts: every column of `:transform_run` except `id`, all optional."@@ -304,9 +271,9 @@(mr/def ::transform-run-cancelation"A TransformRunCancelation as selected from the app DB: every column of `:transform_run_cancelation`."- [:map {:closed true}- [:run_id ms/PositiveInt]- [:time ms/TemporalInstant]])+ [:merge+ ::transform-run-cancelation.update+ [:map {:closed true}]])(mr/def ::transform-run-cancelation.update"What an update (or insert) of a TransformRunCancelation accepts: every column of `:transform_run_cancelation` except `id`, all optional."@@ -316,13 +283,10 @@(mr/def ::transform-tag"A TransformTag as selected from the app DB: every column of `:transform_tag`."- [:map {:closed true}- [:id ms/PositiveInt]- [:name [:or :string mu/localized-string-schema]]- [:entity_id :string]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]- [:built_in_type [:maybe [:or :keyword :string]]]])+ [:merge+ ::transform-tag.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::transform-tag.update"What an update (or insert) of a TransformTag accepts: every column of `:transform_tag` except `id`, all optional."@@ -335,12 +299,10 @@(mr/def ::transform-transform-tag"A TransformTransformTag as selected from the app DB: every column of `:transform_transform_tag`."- [:map {:closed true}- [:id ms/PositiveInt]- [:transform_id ::lib.schema.id/transform]- [:tag_id ms/PositiveInt]- [:entity_id :string]- [:position :int]])+ [:merge+ ::transform-transform-tag.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::transform-transform-tag.update"What an update (or insert) of a TransformTransformTag accepts: every column of `:transform_transform_tag` except `id`, all optional."diff --git a/src/metabase/transforms_base/db.clj b/src/metabase/transforms_base/db.cljindex f4fcb25acf9..d01bf5f99f7 100644--- a/src/metabase/transforms_base/db.clj+++ b/src/metabase/transforms_base/db.clj@@ -58,18 +58,20 @@[database-id :- ::lib.schema.id/databaseschema :- [:maybe :string]table-name :- :string- & conditions :- [:* :some]]+ & conditions :- [:* [:or :keyword :string :int :boolean]]](apply t2/select-one :model/Table :db_id database-id :schema schema :name table-name conditions))(mu/defn table"The Table with `table-id`, or nil."- [table-id :- ::lib.schema.id/table]+ [table-id :- [:maybe ::lib.schema.id/table]](t2/select-one :model/Table :id table-id {:from [(warehouse-schema-overlay/table-query)]}))(mu/defn table-for-transform- "The Table owned by the Transform with `transform-id`, or nil."- [transform-id :- ::lib.schema.id/transform]- (t2/select-one :model/Table :transform_id transform-id {:from [(warehouse-schema-overlay/table-query)]}))+ "The Table owned by the Transform with `transform-id`, or nil. `transform-id` is nil for a transform+ that hasn't been saved yet, in which case there is no owned Table either."+ [transform-id :- [:maybe ::lib.schema.id/transform]]+ (when transform-id+ (t2/select-one :model/Table :transform_id transform-id {:from [(warehouse-schema-overlay/table-query)]})))(mu/defn update-table!"Apply `changes` to the Table with `table-id`."diff --git a/src/metabase/transforms_base/ordering.clj b/src/metabase/transforms_base/ordering.cljindex 63e36a302d9..b8f8768eafb 100644--- a/src/metabase/transforms_base/ordering.clj+++ b/src/metabase/transforms_base/ordering.clj@@ -98,8 +98,14 @@(stored-or-live-deps transform)(catch Throwable _ #{})))+(def ^:private TransformIdAndTargetTable+ "The subset of a Transform row [[output-table-map]] reads: its id and its target Table id, if any."+ [:map {:closed true}+ [:id :metabase.lib.schema.id/transform]+ [:target_table_id {:optional true} [:maybe :metabase.lib.schema.id/table]]])+(mu/defn- output-table-map- [transforms]+ [transforms :- [:sequential TransformIdAndTargetTable]](into {}(keep (fn [{:keys [target_table_id id]}](when target_table_id@@ -164,7 +170,7 @@(logging, metrics, error responses)."[start-ids all-transforms](let [id->xf (u/index-by :id all-transforms)- output-tables (output-table-map all-transforms)+ output-tables (output-table-map (map #(select-keys % [:id :target_table_id]) all-transforms))target-refs (target-ref-map all-transforms)all-ids (into #{} (map :id) all-transforms)](loop [visited {}@@ -262,7 +268,7 @@(map (juxt :id identity))transforms)db-transforms (filter #(= (:source_database_id %) db-id) transforms)- output-tables (output-table-map db-transforms)+ output-tables (output-table-map (map #(select-keys % [:id :target_table_id]) db-transforms))transform-ids (into #{} (map :id) db-transforms)target-refs (target-ref-map transforms)node->children #(->> % transforms-by-id safe-table-dependenciesdiff --git a/src/metabase/transforms_base/query.clj b/src/metabase/transforms_base/query.cljindex a6cfe3618ab..c00700c8ff0 100644--- a/src/metabase/transforms_base/query.clj+++ b/src/metabase/transforms_base/query.clj@@ -7,6 +7,7 @@[clojure.string :as str][metabase.driver :as driver][metabase.driver.util :as driver.u]+ [metabase.indexes.schema :as indexes.schema][metabase.lib.schema.common :as schema.common][metabase.query-processor.compile :as qp.compile][metabase.transforms-base.db :as transforms-base.db]@@ -43,7 +44,7 @@[:output-table [:keyword {:decode/normalize schema.common/normalize-keyword}]];; Declared target indexes. `compile-transform` inlines the ones it can into the CTAS (e.g. Redshift SORTKEY,;; ClickHouse ORDER BY); the rest are created standalone.- [:indexes {:optional true} [:sequential :map]]])+ [:indexes {:optional true} [:sequential ::indexes.schema/index-structured]]])(mr/def ::transform-opts[:mapdiff --git a/src/metabase/transforms_base/schema.clj b/src/metabase/transforms_base/schema.cljindex b7a150fd5ca..5c07895a6d7 100644--- a/src/metabase/transforms_base/schema.clj+++ b/src/metabase/transforms_base/schema.clj@@ -5,6 +5,7 @@The full transforms module (metabase.transforms.schema) extends these withadditional fields like :id (required for scheduled execution)."(:require+ [metabase.indexes.schema :as indexes.schema][metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.metadata :as lib.schema.metadata][metabase.util.malli.registry :as mr]@@ -23,35 +24,81 @@;;; ------------------------------------------------- Transform -------------------------------------------------+(mr/def ::source-incremental-strategy+ "An incremental strategy on a transform's source, as `get-source-range-params` reads it. The full,+ `:type`-dispatched shape (checkpoint/append/merge variants) is owned by `metabase.transforms.schema`;+ this module only reads the checkpoint fields."+ [:map {:closed true}+ [:type {:optional true} [:or :string :keyword]]+ [:checkpoint-filter-field-id {:optional true} ::lib.schema.id/field]+ [:lookback {:optional true} [:maybe [:map {:closed true}+ [:value pos-int?]+ [:unit [:or :string :keyword]]]]]])++(mr/def ::target-incremental-strategy+ "An incremental strategy on a transform's target: append the new rows, or merge them on a unique key."+ [:map {:closed true}+ [:type [:or :string :keyword]]+ [:unique-key {:optional true} [:sequential [:map {:closed true}+ [:name {:optional true} :string]+ [:field-id {:optional true} [:maybe ::lib.schema.id/field]]]]]])+(mr/def ::transform-target"Target specification for a transform. Must include at least :type and :name."- [:map+ [:map {:closed true}[:type :string][:database {:optional true} :int][:schema {:optional true} [:maybe :string]][:name :string]- [:indexes {:optional true} [:sequential :map]]])+ [:indexes {:optional true} [:sequential ::indexes.schema/index-structured]]+ [:target-incremental-strategy {:optional true} ::target-incremental-strategy]+ [:index-request-ids {:optional true} [:sequential pos-int?]]])(mr/def ::transform- "A transform map as expected by execute-base! implementations."- [:map+ "A transform map as expected by execute-base! implementations. The full transforms module+ (metabase.transforms.schema) hydrates and stores more columns on this same map; they're declared here,+ optional, so that richer value can be threaded through the shared execute-base! machinery unchanged."+ [:map {:closed true}[:id {:optional true} pos-int?]- [:source [:map [:type [:or :string :keyword]]]]- [:target ::transform-target]+ [:source [:map {:closed true}+ [:type [:or :string :keyword]]+ [:query {:optional true} [:maybe :metabase.lib-be.schema/maybe-legacy-query]]+ [:body {:optional true} :string]+ [:source-tables {:optional true} [:sequential :metabase.transforms-base.util/source-table-entry]]+ [:source-database {:optional true} :int]+ [:source-incremental-strategy {:optional true} [:maybe ::source-incremental-strategy]]]]+ [:target {:optional true} [:maybe ::transform-target]][:name {:optional true} :string]- [:description {:optional true} [:maybe :string]]])+ [:description {:optional true} [:maybe :string]]+ [:full-incremental-run? {:optional true} :boolean]+ [:entity_id {:optional true} [:maybe :string]]+ [:created_at {:optional true} [:maybe ms/TemporalInstant]]+ [:updated_at {:optional true} [:maybe ms/TemporalInstant]]+ [:source_type {:optional true} [:maybe [:or :keyword :string]]]+ [:creator_id {:optional true} [:maybe ::lib.schema.id/user]]+ [:source_database_id {:optional true} [:maybe ::lib.schema.id/database]]+ [:collection_id {:optional true} [:maybe ::lib.schema.id/collection]]+ [:owner_user_id {:optional true} [:maybe ::lib.schema.id/user]]+ [:owner_email {:optional true} [:maybe :string]]+ [:target_db_id {:optional true} [:maybe ::lib.schema.id/database]]+ [:last_checkpoint_value {:optional true} [:maybe :string]]+ [:target_table_id {:optional true} [:maybe ::lib.schema.id/table]]+ [:table_dependencies {:optional true} [:maybe [:sequential [:or+ [:map {:closed true} [:table ::lib.schema.id/table]]+ [:map {:closed true} [:transform ::lib.schema.id/transform]]]]]]+ [:tag_ids {:optional true} [:maybe [:sequential pos-int?]]]]);;; ----------------------------------------- Source Range Params -----------------------------------------------(mr/def ::checkpoint-bound"A bound (lo or hi) for incremental checkpoint filtering."- [:map- [:value :any]])+ [:map {:closed true}+ [:value [:or number? ms/TemporalInstant]]])(mr/def ::source-range-params"Parameters for incremental range filtering on a source query.Returned by get-source-range-params."- [:map+ [:map {:closed true}[:column ::lib.schema.metadata/column][:checkpoint-filter-field-id ::lib.schema.id/field][:lo {:optional true} [:maybe ::checkpoint-bound]]@@ -63,10 +110,10 @@(mr/def ::execute-base-options"Options map for execute-base! and its implementations."- [:map+ [:map {:closed true}[:cancelled? {:optional true} ifn?][:run-id {:optional true} [:maybe pos-int?]]- [:with-stage-timing-fn {:optional true} ifn?]+ [:with-stage-timing-fn {:optional true} [:maybe ifn?]][:publish-events? {:optional true} :boolean][:message-log {:optional true} [:maybe ::atom]][:cancel-chan {:optional true} [:maybe ::chan]]diff --git a/src/metabase/transforms_base/util.clj b/src/metabase/transforms_base/util.cljindex db6bf00fa42..63c3ac7af38 100644--- a/src/metabase/transforms_base/util.clj+++ b/src/metabase/transforms_base/util.clj@@ -12,6 +12,7 @@[metabase.events.core :as events][metabase.indexes.models.table-index :as table-index][metabase.indexes.reconcile :as reconcile]+ [metabase.indexes.schema :as indexes.schema][metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]@@ -410,7 +411,7 @@:lo values in the source table must be > this :value.:hi values in the source table must be <= this :value.:rows-available count of source rows in (lo, hi] from the same scan; nil if unavailable."- [{:keys [source] :as transform}]+ [{:keys [source] :as transform} :- ::transforms-base.schema/transform](let [{:keys [checkpoint-filter-field-id lookback]} (:source-incremental-strategy source)](validate-incremental-source! transform)(when checkpoint-filter-field-id@@ -482,7 +483,7 @@(mu/defn validate-transform-query :- [:maybe [:map [:error :string]]]"Verifies that a query transform's query can actually be run as is. Returns nil on success and an error map on failure."- [{:keys [source]}]+ [{:keys [source]} :- ::transforms-base.schema/transform](case (keyword (:type source)):query(try@@ -583,19 +584,20 @@;;; ------------------------------------------------- Table DDL -------------------------------------------------(mr/def ::column-definition- [:map+ [:map {:closed true}[:name :string][:type ::lib.schema.common/base-type]- [:nullable? {:optional true} :boolean]])+ [:nullable? {:optional true} :boolean]+ [:database-type {:optional true} [:maybe :string]]])(mr/def ::table-definition- [:map+ [:map {:closed true}[:name :keyword][:columns [:sequential ::column-definition]][:primary-key {:optional true} [:sequential :string]];; Inline indexes to apply at table creation (e.g. a Redshift sortkey). Passed through to `create-table!`;;; drivers that don't inline anything ignore it. Populated from a transform's declared indexes by the manager.- [:indexes {:optional true} [:sequential :map]]])+ [:indexes {:optional true} [:sequential ::indexes.schema/index-structured]]])(mu/defn create-table-from-schema!"Create a table from a table-schema"@@ -797,11 +799,13 @@(and (map? v) (nil? (:table_id v))))(mr/def ::source-table-entry- "A source table entry in the array format. Combines alias with table reference."+ "A source table entry in the array format. Combines alias with table reference. Callers may supply just+ `:table_id` (looked up to fill in `:database_id`/`:schema`/`:table`) or just+ `:database_id`/`:schema`/`:table` (looked up to fill in `:table_id`), so only `:alias` is required."[:map {:closed true}[:alias :string]- [:database_id :int]- [:schema [:maybe :string]]+ [:database_id {:optional true} [:maybe :int]]+ [:schema {:optional true} [:maybe :string]][:table {:optional true} :string][:table_id {:optional true} [:maybe :int]]])@@ -811,7 +815,7 @@For entries with only :database_id/:schema/:table, looks up :table_id.Throws if an integer table ID references a non-existent table.Map refs with non-existent tables get nil table_id (resolved later at execute time)."- [source-tables :- [:sequential [:map [:alias :string]]]]+ [source-tables :- [:sequential ::source-table-entry]](let [;; Entries that have table_id but lack table metadata need lookupneeds-metadata (filter (fn [e] (and (:table_id e) (not (:table e)))) source-tables)int-id->metadata (when (seq needs-metadata)@@ -869,7 +873,13 @@Handles both int values (`{alias: table_id}`) and ref map values (`{alias: {:database_id ...}}`.Accepts both keyword and string keys for alias.Enriches entries with full metadata via [[normalize-source-tables]]."- [m :- [:map-of [:or :string :keyword] [:or :int :map]]]+ [m :- [:map-of :string+ [:or :int+ [:map {:closed true}+ [:database_id {:optional true} [:maybe :int]]+ [:schema {:optional true} [:maybe :string]]+ [:table {:optional true} [:maybe :string]]+ [:table_id {:optional true} [:maybe :int]]]]]](normalize-source-tables(mapv (fn [[alias v]](if (int? v)diff --git a/src/metabase/typed_schemas/db.clj b/src/metabase/typed_schemas/db.cljindex 0152166d75f..a631188f0a5 100644--- a/src/metabase/typed_schemas/db.clj+++ b/src/metabase/typed_schemas/db.clj@@ -50,9 +50,9 @@(t2/select-one [:model/Card :dimensions :dimension_mappings] :id card-id))(mu/defn table-names- "The id, name, and display name of the Tables with `table-ids`."+ "The id, database id, name, and display name of the Tables with `table-ids`."[table-ids :- [:sequential ::lib.schema.id/table]]- (t2/select [:model/Table :id :name :display_name] :id [:in table-ids] {:from [(warehouse-schema-overlay/table-query)]}))+ (t2/select [:model/Table :id :db_id :name :display_name] :id [:in table-ids] {:from [(warehouse-schema-overlay/table-query)]}))(mu/defn model-actions"The id, model id, name, and type of the unarchived non-HTTP Actions of the model Cards with `model-ids`."diff --git a/src/metabase/typed_schemas/scope.clj b/src/metabase/typed_schemas/scope.cljindex 4269b6bfead..40e73d49d78 100644--- a/src/metabase/typed_schemas/scope.clj+++ b/src/metabase/typed_schemas/scope.clj@@ -29,7 +29,8 @@[metabase.collections.models.collection :as collection][metabase.models.interface :as mi][metabase.typed-schemas.db :as typed-schemas.db]- [metabase.util.malli :as mu]))+ [metabase.util.malli :as mu]+ [metabase.util.malli.schema :as ms]))(set! *warn-on-reflection* true)@@ -120,6 +121,19 @@[collection](contains? collection/library-collection-types (:type collection)))+(def ^:private LibraryCollectionRef+ "References a collection by numeric or entity id."+ [:or+ [:map {:closed true} [:id :int]]+ [:map {:closed true} [:entity-id ms/NonBlankString]]])++(def LibraryScopeOptions+ "Options accepted by [[library-scope]]."+ [:map {:closed true}+ [:library-collection-refs {:optional true} [:maybe [:sequential LibraryCollectionRef]]]+ [:include-data-library? {:optional true} [:maybe :boolean]]+ [:include-metric-library? {:optional true} [:maybe :boolean]]])+(defn- library-refs"Returns collection references for explicit library refs plus any includedwell-known library roots."@@ -133,7 +147,7 @@(mu/defn library-scope :- [:maybe LibraryScope]"Resolves library collection refs and include flags into a [[LibraryScope]],or nil when no library scope is requested."- [scope-options]+ [scope-options :- LibraryScopeOptions](when-let [refs (seq (library-refs scope-options))](let [roots (resolve-collection-refs! refs library-collection?)collections (concat roots (collection/descendants-flat-for roots))diff --git a/src/metabase/types/core.cljc b/src/metabase/types/core.cljcindex b30a76b0fe7..7f94dfe9df7 100644--- a/src/metabase/types/core.cljc+++ b/src/metabase/types/core.cljc@@ -386,8 +386,9 @@(mr/def ::snake-cased-type-info"E.g. the version coming back from the app DB as opposed to Lib metadata. This should eventually be considereddeprecated."- [:map- [:base_type :any]])+ [:map {:closed true}+ [:base_type :metabase.lib.schema.common/base-type]+ [:effective_type {:optional true} [:maybe :metabase.lib.schema.common/base-type]]])(mu/defn field-is-type?"True if a Metabase `Field` instance has a temporal base or semantic type, i.e. if this Field represents a valuediff --git a/src/metabase/upload/core.clj b/src/metabase/upload/core.cljindex dd1565adc33..c5ac68c6ae0 100644--- a/src/metabase/upload/core.clj+++ b/src/metabase/upload/core.clj@@ -12,11 +12,12 @@[metabase.upload.dbcurrent-database][metabase.upload.impl+ based-on-upload-input-keyscan-create-upload?create-csv-upload!delete-upload!max-upload-part-countmax-upload-size-bytes- model-hydrate-based-on-upload+ models-based-on-uploadupdate-action-schemaupdate-csv!])diff --git a/src/metabase/upload/db.clj b/src/metabase/upload/db.cljindex 3f0db76c11b..82c963bb4ce 100644--- a/src/metabase/upload/db.clj+++ b/src/metabase/upload/db.clj@@ -121,7 +121,7 @@(mu/defn table"The Table with `table-id`, or nil."- [table-id :- ::lib.schema.id/table]+ [table-id :- [:maybe ::lib.schema.id/table]](t2/select-one :model/Table :id table-id {:from [(warehouse-schema-overlay/table-query)]}))(mu/defn tablesdiff --git a/src/metabase/upload/impl.clj b/src/metabase/upload/impl.cljindex 6fbdfffd52b..6a95fed76a1 100644--- a/src/metabase/upload/impl.clj+++ b/src/metabase/upload/impl.clj@@ -31,6 +31,7 @@[metabase.util :as u][metabase.util.i18n :refer [tru]][metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms][metabase.warehouse-schema.humanization :as warehouse-schema.humanization][metabase.warehouse-schema.models.table :as table]@@ -188,7 +189,7 @@(t/format time-format (strictly-monotonic-now)))))(mu/defn- database-type- [driver+ [driver :- :keywordcolumn-type :- (into [:enum] upload-types/column-types)](let [external-type (keyword "metabase.upload" (name column-type))](driver/upload-type->database-type driver external-type)))@@ -205,9 +206,7 @@(mu/defn table-identifier :- :string"Returns a string that can be used as a table identifier in SQL, including a schema if provided."[{:keys [schema name] :as _table}- :- [:map- [:schema {:optional true} [:maybe :string]]- [:name :string]]]+ :- [:or :metabase.warehouse-schema.schema/table :metabase.warehouse-schema.schema/table.update]](if (str/blank? schema)name(str schema "." name)))@@ -618,7 +617,7 @@- `schema-name`: the name of the schema to create the table in (optional).- `table-prefix`: the prefix to use for the table name (optional)."[{:keys [collection-id filename ^File file db-id schema-name table-prefix]}- :- [:map+ :- [:map {:closed true}[:collection-id [:maybe ms/PositiveInt]][:filename :string][:file (ms/InstanceOfClass File)]@@ -779,7 +778,7 @@(defn- invalidate-cached-models!"Invalidate the model cache and result metadata for all models where `:based_on_upload` resolves to the given table."[table]- ;; NOTE: It is important that this logic is kept in sync with `model-hydrate-based-on-upload`+ ;; NOTE: It is important that this logic is kept in sync with `models-based-on-upload`(when-let [model-ids (->> (upload.db/unarchived-models-for-table (:id table))(filter (comp #{(:id table)} only-table-id))(map :id)@@ -976,7 +975,7 @@This will create an auto-incrementing primary key (auto-pk) column in the table for drivers that supported uploadsbefore auto-pk columns were introduced by metabase#36249, if it does not already exist."[{:keys [filename ^File file table-id action]}- :- [:map+ :- [:map {:closed true}[:table-id ms/PositiveInt][:filename :string][:file (ms/InstanceOfClass File)]@@ -1000,16 +999,26 @@(filter #(can-upload-to-table? (:db %) %))(map :id)))))-(mu/defn model-hydrate-based-on-upload- "Batch hydrates `:based_on_upload` for each item of `models`. Assumes each item of `model` represents a model."- [models :- [:sequential [:map- ;; query_type and dataset_query can be null in tests, so we make them nullable here.- ;; they should never be null in production- [:dataset_query [:maybe ::lib-be.schema/maybe-legacy-or-empty-query]]- [:query_type [:maybe [:or :string :keyword]]]- [:table_id [:maybe ms/PositiveInt]]- ;; is_upload can be provided for an optional optimization- [:is_upload {:optional true} [:maybe :any]]]]]+(mr/def ::based-on-upload-input+ "The columns of a model Card that decide its `:based_on_upload`."+ [:map {:closed true}+ [:id ms/PositiveInt]+ ;; query_type and dataset_query can be null in tests, so we make them nullable here.+ ;; they should never be null in production+ [:dataset_query [:maybe ::lib-be.schema/maybe-legacy-or-empty-query]]+ [:query_type [:maybe [:or :string :keyword]]]+ [:table_id [:maybe ms/PositiveInt]]+ ;; is_upload can be provided for an optional optimization+ [:is_upload {:optional true} [:maybe :boolean]]])++(def based-on-upload-input-keys+ "The keys of a model that [[models-based-on-upload]] reads."+ [:id :dataset_query :query_type :table_id :is_upload])++(mu/defn models-based-on-upload+ "The `:based_on_upload` table id of each of `models` that has one, keyed by model id. Assumes each item of `models`+ represents a model."+ [models :- [:sequential ::based-on-upload-input]](let [table-ids (->> models;; as an optimization when listing collection items (GET /api/collection/items),;; we might already know that the table is not an upload if is_upload=false. We@@ -1018,11 +1027,14 @@(keep :table_id)set)has-uploadable-table? (comp (uploadable-table-ids table-ids) :table_id)]- (for [model models]- ;; NOTE: It is important that this logic is kept in sync with `invalidate-cached-models!`- ;; If not, it will mean that the user could modify the table via a given model's page without seeing it update.- (m/assoc-some model :based_on_upload (when (has-uploadable-table? model)- (only-table-id model))))))+ (into {}+ (keep (fn [model]+ ;; NOTE: It is important that this logic is kept in sync with `invalidate-cached-models!`+ ;; If not, it will mean that the user could modify the table via a given model's page without seeing it update.+ (when-let [table-id (when (has-uploadable-table? model)+ (only-table-id model))]+ [(:id model) table-id])))+ models)))(mi/define-batched-hydration-method based-on-upload:based_on_upload@@ -1037,6 +1049,8 @@Excluding checking the users write permissions for the card, `:based_on_upload` reflects the user'sability to upload to the underlying table through the card."[cards]- (let [id->model (m/index-by :id (model-hydrate-based-on-upload (filter #(= (:type %) :model) cards)))- card->maybe-model (comp id->model :id)]- (map #(or (card->maybe-model %) %) cards)))+ (let [id->table-id (models-based-on-upload (into []+ (comp (filter #(= (:type %) :model))+ (map #(select-keys % based-on-upload-input-keys)))+ cards))]+ (map #(m/assoc-some % :based_on_upload (id->table-id (:id %))) cards)))diff --git a/src/metabase/upload/types.cljc b/src/metabase/upload/types.cljcindex 1a0f8f4b7dd..0a96b6cdc0b 100644--- a/src/metabase/upload/types.cljc+++ b/src/metabase/upload/types.cljc@@ -211,7 +211,9 @@(remove non-inferable-types value-types))))(mu/defn- settings->type->check :- type->check-schema- [{:keys [number-separators] :as _settings}]+ [{:keys [number-separators] :as _settings}+ :- [:map {:closed true}+ [:number-separators [:enum "." ".," ",." ", " ".’"]]]](let [int-string? (regex-matcher (int-regex number-separators))float-or-int? (regex-matcher (float-or-int-regex number-separators))float-string? (regex-matcher (float-regex number-separators))]@@ -268,7 +270,10 @@(mu/defn column-types-from-rows :- [:sequential (into [:enum] column-types)]"Given the types of the existing columns (if there are any), and rows to be added, infer the best supporting types."- [settings existing-types rows]+ [settings :- [:map {:closed true}+ [:number-separators [:enum "." ".," ",." ", " ".’"]]]+ existing-types :- [:maybe [:sequential [:maybe (into [:enum] column-types)]]]+ rows :- [:maybe [:sequential [:sequential [:maybe :string]]]]](let [current-types (mapv #(column-type->abstract % %) existing-types)](->> (reduce (type-relaxer settings) current-types rows)(u/map-all concretize existing-types))))diff --git a/src/metabase/usage_metadata/schema.clj b/src/metabase/usage_metadata/schema.cljindex a953d5d509e..ca7bfc91fe7 100644--- a/src/metabase/usage_metadata/schema.clj+++ b/src/metabase/usage_metadata/schema.clj@@ -115,16 +115,10 @@(mr/def ::source-dimension-daily"A SourceDimensionDaily as selected from the app DB: every column of `:source_dimension_daily`."- [:map {:closed true}- [:id ms/PositiveInt]- [:source_type [:or :keyword :string]]- [:source_id ms/PositiveInt]- [:ownership_mode [:or :keyword :string]]- [:field_id ::lib.schema.id/field]- [:temporal_unit [:maybe [:or :keyword :string]]]- [:binning [:maybe :string]]- [:bucket_date ms/TemporalInstant]- [:count :int]])+ [:merge+ ::source-dimension-daily.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::source-dimension-daily.update"What an update (or insert) of a SourceDimensionDaily accepts: every column of `:source_dimension_daily` except `id`, all optional."@@ -140,16 +134,10 @@(mr/def ::source-dimension-profile-daily"A SourceDimensionProfileDaily as selected from the app DB: every column of `:source_dimension_profile_daily`."- [:map {:closed true}- [:id ms/PositiveInt]- [:source_type [:or :keyword :string]]- [:source_id ms/PositiveInt]- [:field_id ::lib.schema.id/field]- [:source_basis [:or :keyword :string]]- [:observation_type [:or :keyword :string]]- [:observation_value [:maybe :string]]- [:bucket_date ms/TemporalInstant]- [:count :int]])+ [:merge+ ::source-dimension-profile-daily.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::source-dimension-profile-daily.update"What an update (or insert) of a SourceDimensionProfileDaily accepts: every column of `:source_dimension_profile_daily` except `id`, all optional."@@ -165,17 +153,10 @@(mr/def ::source-metric-daily"A SourceMetricDaily as selected from the app DB: every column of `:source_metric_daily`."- [:map {:closed true}- [:id ms/PositiveInt]- [:source_type [:maybe [:or :keyword :string]]]- [:source_id [:maybe ms/PositiveInt]]- [:ownership_mode [:or :keyword :string]]- [:agg_type [:or :keyword :string]]- [:agg_field_id [:maybe ::lib.schema.id/field]]- [:temporal_field_id [:maybe ::lib.schema.id/field]]- [:temporal_unit [:maybe [:or :keyword :string]]]- [:bucket_date ms/TemporalInstant]- [:count :int]])+ [:merge+ ::source-metric-daily.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::source-metric-daily.update"What an update (or insert) of a SourceMetricDaily accepts: every column of `:source_metric_daily` except `id`, all optional."@@ -192,16 +173,10 @@(mr/def ::source-segment-composite-daily"A SourceSegmentCompositeDaily as selected from the app DB: every column of `:source_segment_composite_daily`."- [:map {:closed true}- [:id ms/PositiveInt]- [:source_type [:maybe [:or :keyword :string]]]- [:source_id [:maybe ms/PositiveInt]]- [:ownership_mode [:or :keyword :string]]- [:clause :string]- [:atom_fingerprints :string]- [:atom_count :int]- [:bucket_date ms/TemporalInstant]- [:count :int]])+ [:merge+ ::source-segment-composite-daily.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::source-segment-composite-daily.update"What an update (or insert) of a SourceSegmentCompositeDaily accepts: every column of `:source_segment_composite_daily` except `id`, all optional."@@ -217,15 +192,10 @@(mr/def ::source-segment-daily"A SourceSegmentDaily as selected from the app DB: every column of `:source_segment_daily`."- [:map {:closed true}- [:id ms/PositiveInt]- [:source_type [:maybe [:or :keyword :string]]]- [:source_id [:maybe ms/PositiveInt]]- [:ownership_mode [:or :keyword :string]]- [:field_id [:maybe ::lib.schema.id/field]]- [:predicate :string]- [:bucket_date ms/TemporalInstant]- [:count :int]])+ [:merge+ ::source-segment-daily.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::source-segment-daily.update"What an update (or insert) of a SourceSegmentDaily accepts: every column of `:source_segment_daily` except `id`, all optional."diff --git a/src/metabase/user_key_value/models/user_key_value/types.clj b/src/metabase/user_key_value/models/user_key_value/types.cljindex 2b35134c766..d3e1e658900 100644--- a/src/metabase/user_key_value/models/user_key_value/types.clj+++ b/src/metabase/user_key_value/models/user_key_value/types.clj@@ -3,6 +3,8 @@[clojure.edn :as edn][clojure.java.io :as io][clojure.string :as str]+ [malli.core :as mc]+ [malli.util :as mut][metabase.config.core :as config][metabase.util :as u][metabase.util.files :as u.files]@@ -59,7 +61,7 @@(mu/defn- defnamespace"Declare a new namespace with a schema for the value"[namespace :- ::namespace- schema]+ schema :- [:and any? [:fn {:description "a malli schema"} mc/schema]]](derive! namespace ::registered-namespace)(mr/register! namespace schema))@@ -67,24 +69,63 @@[](descendants hierarchy ::registered-namespace))+(defn- namespace-value-schema+ "The `:value` entry's schema of a registered namespace's own schema (a `:map`, or a `:multi` of `:map`s)."+ [namespace]+ (let [schema (mc/schema (mr/schema namespace))]+ (case (mc/type schema)+ :map (some (fn [[k _opts s]] (when (= k :value) s)) (mc/children schema))+ :multi (let [value-schemas (keep (fn [[_dispatch-value _props branch]]+ (some (fn [[k _opts s]] (when (= k :value) s)) (mc/children branch)))+ (mc/children schema))]+ (when (seq value-schemas)+ (into [:or] value-schemas)))+ nil)))++(defn- known-namespace-value-schemas+ "The union of every registered namespace's `:value` schema, so `::user-key-value` doesn't need to type `:value`+ generically."+ []+ (let [value-schemas (keep namespace-value-schema (known-namespaces))]+ (if (seq value-schemas)+ (into [:or] value-schemas)+ :nil)))+;;; this is just a placeholder so LSP can register the place it lives for jump-to-definition functionality. Actual;;; schema gets created below by [[user-key-value-schema]] and [[update-user-key-value-schema]](mr/def ::user-key-value any?)+(defn- namespace-kvp-schema+ "The schema of a whole key-value pair in `namespace`: its registered schema, merged with the keys every pair has."+ [namespace]+ (let [common [:map {:closed true}+ [:key :string]+ [:namespace :keyword]+ [:expires-at [:maybe ::expires-at]]]+ schema (mc/schema (mr/schema namespace))]+ (case (mc/type schema)+ :map (mut/merge common schema)+ :multi (into [:multi (mc/properties schema)]+ (map (fn [[dispatch-value _props branch]]+ [dispatch-value (mut/merge common branch)]))+ (mc/children schema))+ schema)))+(defn- user-key-value-schema"Build the schema for a `::user-key-value`"[][:and- [:map+ [:map {:closed true}+ [:key :string][:expires-at [:maybe ::expires-at]][:namespace ::namespace][:value {:encode/database json/encode:decode/database #(json/decode % keyword)}- :any]]+ (known-namespace-value-schemas)]](into [:multi{:dispatch :namespace}](map (fn [namespace]- [namespace namespace]))+ [namespace (namespace-kvp-schema namespace)]))(known-namespaces))])(defn- update-user-key-value-schema! []diff --git a/src/metabase/user_key_value/schema.clj b/src/metabase/user_key_value/schema.cljindex 3a5012a5a22..127f8e4f76f 100644--- a/src/metabase/user_key_value/schema.clj+++ b/src/metabase/user_key_value/schema.clj@@ -7,15 +7,10 @@(mr/def ::user-key-value"A UserKeyValue as selected from the app DB: every column of `:user_key_value`."- [:map {:closed true}- [:id ms/PositiveInt]- [:user_id ::lib.schema.id/user]- [:namespace :string]- [:key :string]- [:value [:maybe :string]]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]- [:expires_at [:maybe ms/TemporalInstant]]])+ [:merge+ ::user-key-value.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::user-key-value.update"What an update (or insert) of a UserKeyValue accepts: every column of `:user_key_value` except `id`, all optional."diff --git a/src/metabase/users/db.clj b/src/metabase/users/db.cljindex f6ed073e639..33599b9b80e 100644--- a/src/metabase/users/db.clj+++ b/src/metabase/users/db.clj@@ -169,7 +169,7 @@(mu/defn delete-user-parameter-values!"Delete the UserParameterValues identified by the `:user_id`, `:dashboard_id`, and `:parameter_id` of `parameters`."- [parameters :- [:sequential [:map+ [parameters :- [:sequential [:map {:closed true}[:user_id ::lib.schema.id/user][:dashboard_id ::lib.schema.id/dashboard][:parameter_id :string]]]]diff --git a/src/metabase/users/models/user.clj b/src/metabase/users/models/user.cljindex d510d9f4441..348f583625e 100644--- a/src/metabase/users/models/user.clj+++ b/src/metabase/users/models/user.clj@@ -19,7 +19,6 @@[metabase.util.i18n :as i18n :refer [trs tru]][metabase.util.log :as log][metabase.util.malli :as mu]- [metabase.util.malli.schema :as ms][methodical.core :as methodical][toucan2.core :as t2][toucan2.tools.default-fields :as t2.default-fields]))@@ -367,12 +366,6 @@(declare form-password-reset-url)-(def ^:private Invitor- "Map with info about the admin creating the user, used in the new user notification code"- [:map- [:email ms/Email]- [:first_name [:maybe ms/NonBlankString]]])-(defn serdes-synthesize-user!"Creates a new user with a default password, when deserializing eg. a `:creator_id` field whose email address doesn'tmatch any existing user."@@ -383,10 +376,12 @@"Convenience function for inviting a new `User` and sending them a welcome email.This function will create the user, which will trigger the built-in system eventnotification to send an invite via email."- ([new-user invitor setup?]+ ([new-user :- users.schema/NewUser+ invitor :- ::users.schema/user+ setup? :- :boolean](create-and-invite-user! new-user invitor setup? nil))([new-user :- users.schema/NewUser- invitor :- Invitor+ invitor :- ::users.schema/usersetup? :- :booleaninvite-target :- [:maybe users.schema/InviteTarget]];; create the new user@@ -435,7 +430,10 @@"Adds the `:attributes` key to a user. Only personal users carry attributes; for other user types (API-key, internal)this is always `{}`, so e.g. sandboxed queries made with an API key report a missing user attribute instead ofreading attributes stored on the user row."- [{:keys [login_attributes jwt_attributes] :as user} :- [:map [:type (into [:enum] allowed-user-types)]]]+ [{:keys [login_attributes jwt_attributes] :as user} :- [:merge+ ::users.schema/user+ [:map {:closed true}+ [:type (into [:enum] allowed-user-types)]]]](assoc user :attributes (if (= (:type user) :personal)(merge {} (tenants/login-attributes user) jwt_attributes login_attributes){})))diff --git a/src/metabase/users/models/user_parameter_value.clj b/src/metabase/users/models/user_parameter_value.cljindex 62d2a87f084..2ff95271b01 100644--- a/src/metabase/users/models/user_parameter_value.clj+++ b/src/metabase/users/models/user_parameter_value.clj@@ -41,7 +41,7 @@(let [to-insert (remove #(and (nil? (:value %)) (nil? (:default %))) parameters)](t2/with-transaction [_conn](doseq [batch (partition-all 1000 parameters)]- (users.db/delete-user-parameter-values! batch))+ (users.db/delete-user-parameter-values! (map #(select-keys % [:user_id :dashboard_id :parameter_id]) batch)))(doseq [batch (partition-all 1000 to-insert)](users.db/insert-user-parameter-values! (map #(select-keys % [:user_id :dashboard_id :parameter_id :value]) batch)))))))@@ -70,11 +70,7 @@"Asynchronously delete params with a nil `value` and upsert the rest."[user-id :- pos-int?dashboard-id :- pos-int?- parameters :- [:sequential [:map- [:id [:string {:min 1}]]- ;; TODO -- not sure whether these are optional or not- [:value {:optional true} any?]- [:default {:optional true} some?]]]]+ parameters :- [:sequential :metabase.parameters.schema/parameter-with-value]](when (setting/get :dashboards-save-last-used-parameters)(grouper/submit! @user-parameter-value-queue {:user-id user-id:dashboard-id dashboard-iddiff --git a/src/metabase/users/schema.clj b/src/metabase/users/schema.cljindex a2a3dd15976..a2520c630bf 100644--- a/src/metabase/users/schema.clj+++ b/src/metabase/users/schema.clj@@ -19,9 +19,13 @@(not-any? #(str/starts-with? (name %) "@") (keys attributes)))](deferred-tru "login attribute keys must not start with `@`"))])+(def UserAttributes+ "The combined attributes of a User: its login and JWT attributes plus the system (`@`-prefixed) attributes like its tenant's."+ (ms/string-keyed-map [:ref ::lib.schema.parameter/parameter.value]))+(def NewUser"Required/optionals parameters needed to create a new user (for any backend)"- [:map+ [:map {:closed true}[:first_name {:optional true} [:maybe ms/NonBlankString]][:last_name {:optional true} [:maybe ms/NonBlankString]][:email ms/Email]@@ -31,7 +35,8 @@[:sso_source {:optional true} [:maybe ms/NonBlankString]][:locale {:optional true} [:maybe ms/KeywordOrString]][:type {:optional true} [:maybe ms/KeywordOrString]]- [:tenant_id {:optional true} [:maybe ms/PositiveInt]]])+ [:tenant_id {:optional true} [:maybe ms/PositiveInt]]+ [:is_active {:optional true} [:maybe :boolean]]])(mr/def ::user-group-membership"Group Membership info of a User.@@ -52,22 +57,17 @@(mr/def ::user.settings"The `:settings` column of a User, decoded."- :map)+ ms/UserSettings)(mr/def ::user"A User as selected from the app DB: every column of `:core_user` that the model selects by default, plus `:common_name` added by the model's after-select hook."- [:map {:closed true}- [:id ::lib.schema.id/user]- [:email :string]- [:first_name [:maybe :string]]- [:last_name [:maybe :string]]- [:date_joined ms/TemporalInstant]- [:last_login [:maybe ms/TemporalInstant]]- [:is_superuser :boolean]- [:is_qbnewb :boolean]- [:tenant_id [:maybe ms/PositiveInt]]- [:is_data_analyst :boolean]- [:common_name {:optional true} [:maybe :string]]])+ [:merge+ ::user.update+ [:map {:closed true}+ [:id ::lib.schema.id/user]+ [:common_name {:optional true} [:maybe :string]]+ [:attributes {:optional true} [:maybe UserAttributes]]+ [:user_group_memberships {:optional true} [:sequential [:map {:closed true} [:name :string] [:entity_id :string]]]]]])(mr/def ::user.full"A User as selected from the app DB with every column of `:core_user`, not only the default ones, plus `:common_name` added by the model's after-select hook."@@ -165,12 +165,10 @@(mr/def ::user-parameter-value"A UserParameterValue as selected from the app DB: every column of `:user_parameter_value`."- [:map {:closed true}- [:id ms/PositiveInt]- [:user_id ::lib.schema.id/user]- [:parameter_id :string]- [:value [:maybe ::user-parameter-value.value]]- [:dashboard_id [:maybe ::lib.schema.id/dashboard]]])+ [:merge+ ::user-parameter-value.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::user-parameter-value.update"What an update (or insert) of a UserParameterValue accepts: every column of `:user_parameter_value` except `id`, all optional."diff --git a/src/metabase/users/util.clj b/src/metabase/users/util.cljindex 5694dd7c022..9cff8bbdda9 100644--- a/src/metabase/users/util.clj+++ b/src/metabase/users/util.clj@@ -5,6 +5,7 @@[metabase.api.common :as api][metabase.auth-identity.core :as auth-identity][metabase.config.core :as config]+ [metabase.lib.schema.id :as lib.schema.id][metabase.notification.core :as notification][metabase.permissions.core :as perms][metabase.premium-features.core :as premium-features]@@ -15,6 +16,7 @@[metabase.util :as u][metabase.util.i18n :refer [tru]][metabase.util.malli :as mu]+ [metabase.util.malli.schema :as ms][toucan2.core :as t2]))(defn check-self-or-superuser@@ -36,9 +38,9 @@(mu/defn maybe-set-user-group-memberships!"Implementation for `POST /api/user` and friends; set the PermissionsGroupMemberships for a `user-or-id`."- [user-or-id+ [user-or-id :- ::lib.schema.id/usernew-user-group-memberships :- [:maybe [:sequential ::users.schema/user-group-membership]]- & [is-superuser?]]+ & [is-superuser?] :- [:* [:maybe :boolean]]](when new-user-group-memberships;; if someone passed in both `:is_superuser` and `:group_ids`, make sure the whether the admin group is in group_ids;; agrees with is_superuser -- don't want to have ambiguous behavior@@ -64,8 +66,16 @@sourcetenant-idinvite-target]- :as attributes} :- [:map- [:source {:optional true, :default :admin} [:enum :setup :admin]]]]+ :as attributes} :- [:map {:closed true}+ [:first-name {:optional true} [:maybe ms/NonBlankString]]+ [:last-name {:optional true} [:maybe ms/NonBlankString]]+ [:email ms/Email]+ [:password {:optional true} [:maybe ms/NonBlankString]]+ [:user-group-memberships {:optional true} [:maybe [:sequential ::users.schema/user-group-membership]]]+ [:login-attributes {:optional true} [:maybe users.schema/LoginAttributes]]+ [:source {:optional true, :default :admin} [:enum :setup :admin]]+ [:tenant-id {:optional true} [:maybe ms/PositiveInt]]+ [:invite-target {:optional true} [:maybe users.schema/InviteTarget]]]](api/check-superuser)(api/check-400 (not (users.db/user-email-exists? (u/lower-case-en email))){:errors {:email (tru "Email address already in use.")}diff --git a/src/metabase/users_rest/api.clj b/src/metabase/users_rest/api.cljindex f93aa156bcb..dd28122deeb 100644--- a/src/metabase/users_rest/api.clj+++ b/src/metabase/users_rest/api.clj@@ -86,7 +86,10 @@(mu/defn- combine :- CombinedAttributes"Combines user, tenant, and system attributes. User can override "- [attributes :- [:map-of :keyword [:maybe SimpleAttributes]]+ [attributes :- [:map {:closed true}+ [:jwt {:optional true} [:maybe SimpleAttributes]]+ [:tenant {:optional true} [:maybe SimpleAttributes]]+ [:user {:optional true} [:maybe SimpleAttributes]]]system :- [:maybe SystemAttributes]](letfn [(value-map [s f vs] (into {}(for [[k v] vs]diff --git a/src/metabase/util.cljc b/src/metabase/util.cljcindex cb84a081233..1a0f88e49bf 100644--- a/src/metabase/util.cljc+++ b/src/metabase/util.cljc@@ -547,7 +547,7 @@replaced with underscores in order to support languages that don't use the Latin alphabet; see metabase#3818).Optionally specify `:max-length` which will truncate the slug after that many characters."- (^String [^String s]+ (^String [^String s :- [:maybe :string]](slugify s {}))(^String [s :- [:maybe :string]{:keys [max-length unicode?]} :- [:maybediff --git a/src/metabase/util/cron.clj b/src/metabase/util/cron.cljindex e68b4b6e21b..def6285e16b 100644--- a/src/metabase/util/cron.clj+++ b/src/metabase/util/cron.clj@@ -66,7 +66,15 @@(mu/defn- cron-string :- CronScheduleString"Build a cron string from key-value pair parts."- [{:keys [seconds minutes hours day-of-month month day-of-week year]}]+ [{:keys [seconds minutes hours day-of-month month day-of-week year]}+ :- [:map {:closed true}+ [:seconds {:optional true} [:maybe [:or :string :int]]]+ [:minutes {:optional true} [:maybe [:or :string :int]]]+ [:hours {:optional true} [:maybe [:or :string :int]]]+ [:day-of-month {:optional true} [:maybe [:or :string :int]]]+ [:month {:optional true} [:maybe [:or :string :int]]]+ [:day-of-week {:optional true} [:maybe [:or :string :int]]]+ [:year {:optional true} [:maybe [:or :string :int]]]]](str/join " " [(or seconds "0")(or minutes "0")(or hours "*")diff --git a/src/metabase/util/date_2.clj b/src/metabase/util/date_2.cljindex 07132fd86be..a04dad6bbea 100644--- a/src/metabase/util/date_2.clj+++ b/src/metabase/util/date_2.clj@@ -181,7 +181,8 @@(add (t/zoned-date-time \"2019-11-05T15:44-08:00[US/Pacific]\") :month 2)->(t/zoned-date-time \"2020-01-05T15:44-08:00[US/Pacific]\")"- ([unit amount]+ ([unit :- (into [:enum] add-units)+ amount :- [:maybe :int]](add (t/zoned-date-time) unit amount))([t :- TemporalInstance@@ -248,7 +249,7 @@Values are returned as numbers (currently, always and integers, but this may change if we add support for`:fraction-of-second` in the future.)"- ([unit]+ ([unit :- (into [:enum] (conj extract-units :day-of-week-iso))](extract (t/zoned-date-time) unit))([t :- TemporalInstance@@ -329,7 +330,7 @@"Truncate a temporal value `t` to the beginning of `unit`, e.g. `:hour` or `:day`. Not all truncation units aresupported on all subclasses of `Temporal` — for example, you can't truncate a `LocalTime` to `:month`, for obviousreasons."- ([unit]+ ([unit :- (into [:enum] truncate-units)](truncate (t/zoned-date-time) unit))([^Temporal t :- TemporalInstance@@ -354,7 +355,7 @@(group-by #(bucket % :quarter-of-year) (map t/local-date [\"2019-01-01\" \"2019-01-02\" \"2019-01-04\"]));; -> {1 [(t/local-date \"2019-01-01\") (t/local-date \"2019-01-02\")], 2 [(t/local-date \"2019-01-04\")]}"- ([unit]+ ([unit :- (into [:enum] cat [extract-units truncate-units])](bucket (t/zoned-date-time) unit))([t :- TemporalInstance@@ -375,10 +376,11 @@->{:start (t/zoned-date-time \"2019-10-27T00:00Z[UTC]\"):end (t/zoned-date-time \"2019-11-03T00:00Z[UTC]\")}"- ([unit]+ ([unit :- (into [:enum] add-units)](range (t/zoned-date-time) unit))- ([t unit]+ ([t :- TemporalInstance+ unit :- (into [:enum] add-units)](range t unit nil))([t :- TemporalInstance@@ -386,7 +388,11 @@{:keys [start end resolution]:or {start :inclusiveend :exclusive- resolution :millisecond}}]+ resolution :millisecond}}+ :- [:maybe [:map {:closed true}+ [:start {:optional true} [:enum :inclusive :exclusive]]+ [:end {:optional true} [:enum :inclusive :exclusive]]+ [:resolution {:optional true} (into [:enum] add-units)]]]](let [t (truncate t unit)]{:start (case start:inclusive tdiff --git a/src/metabase/util/date_2/parse.clj b/src/metabase/util/date_2/parse.cljindex 25d42bc9e9b..9e1e4adca90 100644--- a/src/metabase/util/date_2/parse.clj+++ b/src/metabase/util/date_2/parse.clj@@ -46,6 +46,11 @@{:error/message "Instance of a java.time.temporal.Temporal"}(partial instance? Temporal)])+(def ^:private InstanceOfDateTimeFormatter+ [:fn+ {:error/message "Instance of a java.time.format.DateTimeFormatter"}+ (partial instance? DateTimeFormatter)])+(def ^:private utc-zone-region (t/zone-id "UTC"))(defn- try-parse-as-iso-timestamp@@ -63,8 +68,8 @@(mu/defn parse-with-formatter :- [:maybe InstanceOfTemporal]"Parse a String with a DateTimeFormatter, returning an appropriate instance of an `java.time` temporal class."- [formattr- s :- [:maybe :string]]+ [formattr :- InstanceOfDateTimeFormatter+ s :- [:maybe :string]]{:pre [((some-fn string? nil?) s)]}(when-not (str/blank? s)(let [formattr (t/formatter formattr)diff --git a/src/metabase/util/honey_sql_2.clj b/src/metabase/util/honey_sql_2.cljindex 4cd896e1c32..031c2fe3f3a 100644--- a/src/metabase/util/honey_sql_2.clj+++ b/src/metabase/util/honey_sql_2.clj@@ -9,6 +9,7 @@[honey.sql.protocols :as sql.protocols][metabase.util :as u][metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms][potemkin.types :as p.types])(:import@@ -152,6 +153,19 @@;; for [[quoted-cast]]:type-name])+(mr/def ::expr+ "A Honey SQL 2 expression: a literal value, a column/table identifier keyword, or a vector-form SQL clause+ (recursively, e.g. a function call, cast, or tagged form like [[identifier]] or a `TypedHoneySQLForm`)."+ [:or+ :keyword+ :string+ number?+ :boolean+ nil?+ ms/TemporalInstant+ [:fn {:error/message "::h2x/typed Honey SQL form"} (fn [x] (and (vector? x) (= (first x) ::typed)))]+ [:sequential [:ref ::expr]]])+(defn identifier?"Whether `x` is a valid `::identifier`."[x]@@ -224,7 +238,7 @@this won't handle wacky cases like three single quotes in a row.DON'T USE `LITERAL` FOR THINGS THAT MIGHT BE WACKY (USER INPUT). Only use it for things that are hardcoded."- [s]+ [s :- [:or :string :keyword]][::literal (u/qualified-name s)])(defn- format-at-time-zone [_tag [expr zone]]@@ -272,10 +286,17 @@(fn [s](= s (u/lower-case-en s)))]]]])+(def ^:private TypeInfo+ "Type info for a `TypedHoneySQLForm` before [[normalize-type-info]], open to the namespaced keys drivers own (e.g. `:metabase.driver.postgres/target-timezone`)."+ [:map {:closed false, ::mr/deliberately-open true}+ [:database-type {:optional true} [:maybe ms/KeywordOrString]]+ [:base-type {:optional true} [:maybe :keyword]]+ [:effective-type {:optional true} [:maybe :keyword]]])+(mu/defn- normalize-type-info :- NormalizedTypeInfo"Normalize the values in the `type-info` for a `TypedHoneySQLForm` for easy comparisons (e.g., normalize`:database-type` to a lower-case string)."- [type-info]+ [type-info :- [:maybe TypeInfo]](cond-> type-info(:database-type type-info)(update :database-type (comp u/lower-case-en name))))@@ -371,7 +392,8 @@(with-database-type-info :field \"text\");; -> [::typed :field \"text\"]"{:style/indent [:form]}- [honeysql-form db-type :- [:maybe ms/KeywordOrString]]+ [honeysql-form :- ::honeysql-expr+ db-type :- [:maybe ms/KeywordOrString]](if (some? db-type)(with-type-info honeysql-form {:database-type db-type})(unwrap-typed-honeysql-form honeysql-form)))@@ -379,6 +401,24 @@(def ^:private TypedExpression[:fn {:error/message "::h2x/typed Honey SQL form"} typed?])+(mr/def ::honeysql-clause-opts+ "A map inside a Honey SQL 2 clause (a window spec, a subquery, ...), whose keys the Honey SQL library and the clauses drivers register own."+ [:map {:closed false, ::mr/deliberately-open true}])++(mr/def ::honeysql-expr+ "A Honey SQL 2 expression: a literal value, a column/identifier keyword, a `TypedHoneySQLForm` wrapping another+ expression, or a clause vector whose args are themselves Honey SQL expressions (or a clause options map, e.g.+ `:over`'s window spec)."+ [:or+ :string+ :keyword+ number?+ :boolean+ nil?+ ms/TemporalInstant+ [:fn {:error/message "::h2x/typed Honey SQL form"} typed?]+ [:sequential [:or [:ref ::honeysql-expr] ::honeysql-clause-opts]]])+(def ^:private raw-cast-type-name-re#"(?i)[a-z][a-z0-9_ ]*(?:\(\d+(?:, ?\d+)?\))?")@@ -392,7 +432,8 @@(mu/defn cast :- TypedExpression"Generate a statement like `cast(expr AS sql-type)`. Returns a typed HoneySQL form."- [sql-type expr]+ [sql-type :- ms/KeywordOrString+ expr :- ::honeysql-expr](-> (if (raw-type-name? sql-type)[:cast expr ^:allow-raw-sql [:raw (name sql-type)]][:cast expr (identifier :type-name (name sql-type))])@@ -400,7 +441,8 @@(mu/defn maybe-cast :- TypedExpression"Cast `expr` to `sql-type`, unless `expr` is typed and already of that type. Returns a typed HoneySQL form."- [sql-type expr]+ [sql-type :- [:maybe ms/KeywordOrString]+ expr :- ::honeysql-expr](if (or (nil? sql-type)(is-of-type? expr sql-type))exprdiff --git a/src/metabase/util/malli.cljc b/src/metabase/util/malli.cljcindex b3f78873a71..1b40e828315 100644--- a/src/metabase/util/malli.cljc+++ b/src/metabase/util/malli.cljc@@ -44,10 +44,31 @@;; The compiler seems to just inline the translated strings with no annotation or wrapping.:cljs :string))+(mr/def ::json-schema-doc+ "A hand-authored JSON Schema document: only ever echoed to an LLM or OpenAPI consumer, never read by key."+ [:map {:closed true}+ [:type {:optional true} :string]+ [:const {:optional true} :string]+ [:description {:optional true} :string]+ [:required {:optional true} [:sequential :string]]+ [:minItems {:optional true} :int]+ [:maxItems {:optional true} :int]+ [:items {:optional true} [:ref ::json-schema-doc]]+ [:properties {:optional true} [:map-of :string [:ref ::json-schema-doc]]]])++(def ^:private SchemaProperties+ "The malli schema properties [[with]] is actually asked to attach."+ [:map {:closed true}+ [:description {:optional true} [:or :string localized-string-schema]]+ [:decode/api {:optional true} ifn?]+ [:json-schema {:optional true} ::json-schema-doc]+ [::mr/deliberately-open {:optional true} :boolean]])+(metabase.util.malli/defn with"Update a malli schema with an arbitrary map of properties"{:style/indent [:form]}- [mschema props]+ [mschema :- Schema+ props :- SchemaProperties](mut/update-properties (mc/schema mschema) merge props));; Kondo gets confused by :refer [defn] on this, so it's referenced fully qualified.@@ -62,7 +83,7 @@{:style/indent [:form]}([mschema :- Schema error-message :- localized-string-schema](with-api-error-message mschema error-message error-message))- ([mschema :- :any+ ([mschema :- Schemadescription-message :- localized-string-schemaspecific-error-message :- localized-string-schema](mut/update-properties (mc/schema mschema) assoc@@ -118,7 +139,7 @@:cljs `(-defmethod-cljs ~multifn ~dispatch-value ~@fn-tail))))#?(:clj- (defn validate-throw+ (core/defn validate-throw"Returns the value if it matches the schema, else throw an exception."[schema-or-validator value](let [is-validator? (fn? schema-or-validator)]diff --git a/src/metabase/util/malli/closed_schemas.clj b/src/metabase/util/malli/closed_schemas.cljnew file mode 100644index 00000000000..1e44dfe4897--- /dev/null+++ b/src/metabase/util/malli/closed_schemas.clj@@ -0,0 +1,200 @@+(ns metabase.util.malli.closed-schemas+ "Find the schemas reachable from a schema that do not declare their shape: open maps, keyword-keyed `:map-of`s and+ `:any`.++ Accepted instead: a map with a `::mc/default` entry whose schema is a map schema (and is walked), or any+ `::mc/default` entry inside a conjunct of an `:and`, `:any` as a conjunct of an `:and`,+ and the schemas marked `::mr/deliberately-open`. Registry schemas are walked once per `visited` set, except the ones of a+ library."+ (:require+ [clojure.string :as str]+ [malli.core :as mc]+ [metabase.config.core :as config]+ [metabase.util.malli.registry :as mr]))++(set! *warn-on-reflection* true)++(def ^:dynamic *enabled*+ "Whether the checks in this namespace do anything; off in prod."+ (not config/is-prod?))++(defn- ref-name+ "The registry key a schema refers to, for `[:ref ::foo]` and for a bare `::foo` reference alike."+ [schema]+ (if (= (mc/type schema) :ref)+ (let [child (first (mc/children schema))]+ (when (qualified-keyword? child) child))+ (when (mc/-ref-schema? schema)+ (let [form (mc/form schema)]+ (when (qualified-keyword? form) form)))))++(defn- deref-safe [schema]+ (let [dereffed (try (mc/deref schema) (catch Throwable _ nil))]+ (when-not (identical? dereffed schema)+ dereffed)))++(def ^:private open-map-of-key-schemas+ "`:map-of` key schemas that make the map an open bag of arbitrary keys."+ #{:keyword 'keyword? :qualified-keyword 'qualified-keyword? :simple-keyword 'simple-keyword?+ :any :some 'any? 'some?})++(defn- open-map-of-key?+ "Whether a `:map-of` key schema accepts arbitrary keywords, alone or as an alternative."+ [key-schema]+ (let [key-schema (try (mc/deref-all (mc/schema key-schema)) (catch Throwable _ nil))]+ (boolean+ (when key-schema+ (or (contains? open-map-of-key-schemas (mc/form key-schema))+ (contains? open-map-of-key-schemas (mc/type key-schema))+ (and (contains? #{:or :and :maybe :schema} (mc/type key-schema))+ (some open-map-of-key? (mc/children key-schema))))))))++(def ^:private open-map-schemas+ "Predicate schemas that accept a map with any keys."+ #{'map?})++(def ^:private any-schemas+ "Schemas that let a value of any shape through."+ #{:any 'any? :some 'some?})++(def ^:private value-schemas+ "Schemas whose children are not walked: values, and the signatures of function schemas, which validate as `ifn?`."+ #{:= :not= :enum :fn :re :=> :-> :function})++(defn- external-registry-key?+ "Whether `registry-key` belongs to a namespace outside Metabase, such as a library's schemas."+ [registry-key]+ (let [ns-name (namespace registry-key)]+ (and (not (str/starts-with? ns-name "metabase"))+ (some? (find-ns (symbol ns-name))))))++(defn- deliberately-open? [schema]+ (true? (::mr/deliberately-open (mc/properties schema))))++(defn- default-entry? [child]+ (and (vector? child) (= (first child) ::mc/default)))++(defn- declaring-default-entry?+ "Whether `child` is a `::mc/default` entry whose schema is a map schema for the other keys: a deliberately open map,+ a `:map-of`, or a closed map."+ [child]+ (when (default-entry? child)+ (when-let [default (try (mc/deref-all (mc/schema (peek child))) (catch Throwable _ nil))]+ (or (deliberately-open? default)+ (contains? #{:map-of :multi} (mc/type default))+ (and (= :map (mc/type default))+ (true? (:closed (mc/properties default))))))))++(defn- finding [schema trail kind]+ {:trail trail, :kind kind, :form (mc/form schema)})++(defn- walk-schema!+ "Walk `schema`, adding to `findings`; `conjunct?` is true inside a conjunct of an `:and`, where `:any` is allowed."+ [schema trail {:keys [visited findings any? skip-entry?], :as options} conjunct?]+ (when-let [schema (try (mc/schema schema) (catch Throwable _ nil))]+ (let [schema-type (mc/type schema)+ registry-key (ref-name schema)]+ (cond+ registry-key+ (when-not (or (contains? @visited registry-key)+ (external-registry-key? registry-key))+ (swap! visited conj registry-key)+ (when-let [dereffed (deref-safe schema)]+ (walk-schema! dereffed (conj trail registry-key) options conjunct?)))++ (deliberately-open? schema)+ nil++ :else+ (do+ (cond+ (or (and (= schema-type :map)+ (not (true? (:closed (mc/properties schema))))+ (not (some (if conjunct? default-entry? declaring-default-entry?) (mc/children schema))))+ (contains? open-map-schemas schema-type))+ (swap! findings conj (finding schema trail :open-map))++ (and (= schema-type :map-of)+ (open-map-of-key? (first (mc/children schema))))+ (swap! findings conj (finding schema trail :keyword-keyed-map-of))++ (and any? (contains? any-schemas schema-type) (not conjunct?))+ (swap! findings conj (finding schema trail :any)))+ (cond+ (mc/-ref-schema? schema)+ (when-let [dereffed (deref-safe schema)]+ (walk-schema! dereffed trail options conjunct?))++ (contains? value-schemas schema-type)+ nil++ :else+ (let [conjunct? (case schema-type+ :and true+ (:schema :multi) conjunct?+ (:sequential :vector :set :tuple :maybe :or) false+ conjunct?)]+ (doseq [[i child] (map-indexed vector (mc/children schema))]+ (cond+ (and (vector? child) (= 3 (count child)) (mc/schema? (nth child 2)))+ (when-not (and (= schema-type :map)+ (not (default-entry? child))+ (skip-entry? schema (first child)))+ (walk-schema! (nth child 2) (conj trail (first child)) options+ (if (= schema-type :map)+ (and conjunct? (default-entry? child))+ conjunct?)))++ (= schema-type :map-of)+ (walk-schema! child (conj trail i) options (and conjunct? (zero? i)))++ (or (mc/schema? child) (vector? child) (qualified-keyword? child))+ (walk-schema! child (conj trail i) options conjunct?))))))))))++(defn findings+ "Every open map, keyword-keyed `:map-of` and, when `:any?`, `:any` reachable from `schema`, each with the `:trail`+ of keys leading to it.++ Options: `:visited`, an atom of registry keys to skip and add to; `:any?`, default true; `:skip-entry?`, a+ `(fn [map-schema k])` for map entries not to walk."+ ([schema]+ (findings schema {}))+ ([schema {:keys [visited any? skip-entry?], :or {any? true, skip-entry? (constantly false)}}]+ (let [found (atom [])]+ (walk-schema! schema [] {:visited (or visited (atom #{}))+ :findings found+ :any? any?+ :skip-entry? skip-entry?}+ false)+ @found)))++(defn describe-findings+ "A line per finding, for an error message."+ [found]+ (str/join "\n" (for [{:keys [trail kind form]} found]+ (format " %s at %s: %s"+ (case kind+ :open-map "open map"+ :keyword-keyed-map-of "keyword-keyed map-of"+ :any ":any")+ (if (seq trail) (pr-str trail) "the root")+ (pr-str form)))))++(def ^:private visited-arg-registry-keys+ "Registry keys [[check-args!]] has already walked."+ (atom #{}))++(defn check-args!+ "Throw when an argument schema of `fn-name`, or an argument without one, can reach a schema that does not declare+ its shape."+ [fn-name arg-schemas]+ (when *enabled*+ (when-let [found (not-empty (into []+ (mapcat #(findings % {:visited visited-arg-registry-keys}))+ arg-schemas))]+ (throw (ex-info (format (str "The arguments of %s reach schemas that do not declare their shape. Give every"+ " argument a schema, close every map with {:closed true}, key a map-of by"+ " :string, and give :any a type:\n%s")+ fn-name+ (describe-findings found))+ {:fn-name fn-name, :findings found})))))diff --git a/src/metabase/util/malli/defn.clj b/src/metabase/util/malli/defn.cljindex 3e6dbb8c83e..396904f5c06 100644--- a/src/metabase/util/malli/defn.clj+++ b/src/metabase/util/malli/defn.clj@@ -122,7 +122,8 @@~(if instrument?(macros/case:clj (let [error-context {:fn-name (list 'quote fn-name)}]- (mu.fn/instrumented-fn-form error-context :clj parsed cosmetic-name))+ (mu.fn/instrumented-fn-form error-context :clj parsed cosmetic-name+ (symbol (str *ns*) (str fn-name)))):cljs (mu.fn/deparameterized-fn-form :cljs parsed cosmetic-name))(mu.fn/deparameterized-fn-form (macros/case :clj :clj, :cljs :cljs) parsed)))))diff --git a/src/metabase/util/malli/fn.clj b/src/metabase/util/malli/fn.cljindex 8daa4a74b3d..1415b37b1e0 100644--- a/src/metabase/util/malli/fn.clj+++ b/src/metabase/util/malli/fn.clj@@ -8,6 +8,7 @@[metabase.config.core :as config][metabase.util.i18n :as i18n][metabase.util.log :as log]+ [metabase.util.malli.closed-schemas :as mu.closed-schemas][metabase.util.malli.humanize :as mu.humanize][metabase.util.malli.registry :as mr][net.cgrand.macrovich :as macros]))@@ -401,11 +402,18 @@return an unevaluated instrumented [[fn]] form like(mc/-instrument {:schema [:=> [:cat :int :any] :any]}- (fn [x y] (+ 1 2)))"- [error-context lang parsed & [fn-name]]+ (fn [x y] (+ 1 2)))++ With `checked-name`, the form also checks with [[mu.closed-schemas/check-args!]] that every argument map is closed."+ [error-context lang parsed & [fn-name checked-name]](let [[fn-schema captured] (capture-schemas (fn-schema parsed))]`(let [~'&f ~(deparameterized-fn-form lang parsed fn-name)~@(into [] cat captured)]+ ~@(when checked-name+ [`(mu.closed-schemas/check-args! '~checked-name+ ~(case (first fn-schema)+ :=> [(second fn-schema)]+ :function (mapv second (rest fn-schema))))])(core/fn ~'mufn ~@(instrumented-fn-tail error-context fn-schema)))));; ------------------------------ Skipping Namespace Enforcement in prod ------------------------------diff --git a/src/metabase/util/malli/schema.clj b/src/metabase/util/malli/schema.cljindex f792fcbb4dc..aa248dc606c 100644--- a/src/metabase/util/malli/schema.clj+++ b/src/metabase/util/malli/schema.clj@@ -228,11 +228,86 @@are not knowable here."(open-map "database connection details"))+(defn- open-json+ "A JSON value whose objects are deliberately open maps: a scalar, a sequence of such values, or an open map."+ [registry-key description]+ [:schema+ {:registry {registry-key [:or+ :string+ number?+ :boolean+ :nil+ :keyword+ [:sequential [:ref registry-key]]+ (open-map description)]}}+ [:ref registry-key]])++(def RingRequestBody+ "The parsed body of a Ring request, before the schema of the endpoint that received it decodes and validates it."+ (open-json ::ring-request-body "request body"))++(def RingRequestParams+ "The route, query or form params of a Ring request, before the endpoint's own schema decodes and validates them."+ (open-map "request params"))++(def RingResponseBody+ "What an endpoint returns, before its response schema encodes it."+ (open-json ::ring-response-body "response body"))++(def VisualizationSettingsValue+ "A value nested inside [[VisualizationSettings]], whose shape the frontend owns."+ (open-json ::visualization-settings-value "visualization settings value"))++(def HttpRequestBody+ "The body the test HTTP client sends, as written by the test calling the endpoint: a JSON-shaped value, or (for a+ multipart upload, where a key can repeat) a sequence of `[key value]` pairs whose value may be a byte array,+ `File`, or `InputStream` for a file part."+ [:or+ (open-json ::http-request-body "HTTP request body")+ [:sequential [:tuple [:or :string :keyword]+ [:or :string number? :boolean :nil :keyword+ bytes?+ (InstanceOfClass java.io.File)+ (InstanceOfClass java.io.InputStream)]]]])++(def HttpQueryParams+ "The query parameters the test HTTP client sends, as written by the test calling the endpoint."+ (open-map "HTTP query params"))++(def JSONSchemaLiteral+ "A literal value inside a JSON Schema document (`default`, `const`, `enum` or `examples`)."+ (open-json ::json-schema-literal "JSON Schema literal"))++(def ExceptionData+ "The `ex-data` of an exception, whose keys belong to whatever code threw it."+ (open-map "exception data"))++(def AuditLogDetails+ "The `:details` of an audit log event, whose keys depend on the event's topic."+ (open-map "audit log details"))++(def JWTClaims+ "The claims of a decoded JWT, whose keys belong to the identity provider."+ (open-map "JWT claims"))++(def SAMLAttributes+ "The attributes of a SAML assertion, whose keys belong to the identity provider."+ (open-map "SAML attributes"))++(def TenantAttributes+ "The attributes of a tenant, whose keys are defined by the admin."+ (open-map "tenant attributes"))+(def DatabaseSettings"A Database's `:settings`: database-local settings, whose keys are owned by the settings registry rather than by thisschema. Not for any other bag of settings."(open-map "database settings"))+(def UserSettings+ "A User's `:settings`: user-local settings, whose keys are owned by the settings registry rather than by this+ schema. Not for any other bag of settings."+ (open-map "user settings"))+(defn string-keyed-map"Schema for a JSON object whose keys are not ours to declare, as a `:map-of` string keys to `value-schema`.Normalizing stringifies its keys -- a request on its way in, or a keywordized JSON column read back from the@@ -338,8 +413,11 @@(def FieldValue"One value of a Field: a JSON scalar as kept in the `field_values.values` and `human_readable_values` columns, or, on- the way there, a UUID or `java.time` object as the query returned it."- [:maybe [:or :string number? :boolean uuid? (InstanceOfClass java.time.temporal.Temporal)]])+ the way there, a UUID, a `java.time` object, or a legacy `java.util.Date` (`java.sql.Date`/`Timestamp`) as the+ query returned it."+ [:maybe [:or :string number? :boolean uuid?+ (InstanceOfClass java.time.temporal.Temporal)+ (InstanceOfClass java.util.Date)]])(def RemappedFieldValue"Has two components:@@ -371,7 +449,7 @@embed author's, so the map is string-keyed on its way in; it is stored as JSON and read back keywordized like everyother JSON column."(mu/with-api-error-message- [:maybe (string-keyed-map [:enum "disabled" "enabled" "locked"])]+ [:maybe (string-keyed-map [:enum "disabled" "enabled" "locked" :disabled :enabled :locked])](deferred-tru "value must be a valid embedding params map.")))(def ValidLocalediff --git a/src/metabase/util/queue.clj b/src/metabase/util/queue.cljindex 9d7c0fb8454..b44ef54644b 100644--- a/src/metabase/util/queue.clj+++ b/src/metabase/util/queue.clj@@ -113,11 +113,13 @@(when-let [fst (.poll queue max-first-ms TimeUnit/MILLISECONDS)](take-batch* queue max-batch-messages max-next-ms [(if (instance? DelayQueue queue) (:value fst) fst)]))))-(mr/def ::listener-options [:map [:success-handler {:optional true} [:=> [:cat :any :double :string] :any]- :err-handler {:optional true} [:=> [:cat [:fn (ms/InstanceOfClass Throwable) :string]] :any]- :pool-size {:optional true} number?- :max-batch-messages {:optional true} number?- :max-next-ms {:optional true} number?]])+(mr/def ::listener-options+ [:map {:closed true}+ [:success-handler {:optional true} fn?]+ [:err-handler {:optional true} fn?]+ [:pool-size {:optional true} number?]+ [:max-batch-messages {:optional true} number?]+ [:max-next-ms {:optional true} number?]])(defonce^:private@@ -132,7 +134,7 @@(mu/defn- listener-thread [listener-name :- :stringqueue :- (ms/InstanceOfClass BlockingQueue)- handler :- [:=> [:cat [:sequential :any]] :any]+ handler :- fn?{:keys [success-handler err-handler max-batch-messages max-next-ms]} :- ::listener-options](log/debugf "Thread for listener %s started" listener-name)(while true@@ -192,7 +194,7 @@- max-next-ms: Max number of ms to wait for each additional message before calling the handler. Default 100"[listener-name :- :stringqueue :- (ms/InstanceOfClass BlockingQueue)- handler :- [:=> [:cat [:sequential :any]] :any]+ handler :- fn?{:keys [success-handlererr-handlerpool-sizediff --git a/src/metabase/util/retry.clj b/src/metabase/util/retry.cljindex 3d824b03c95..1ff74562a47 100644--- a/src/metabase/util/retry.clj+++ b/src/metabase/util/retry.clj@@ -10,12 +10,13 @@(set! *warn-on-reflection* true)(mr/def ::retry-config- [:map+ [:map {:closed true}[:max-retries :int][:initial-interval-millis :int][:multiplier :float][:jitter-factor :float]- [:max-interval-millis :int]])+ [:max-interval-millis :int]+ [:delay-ms {:optional true} :int]])(mr/def ::retry-overrides(mut/optional-keys [:ref ::retry-config]))diff --git a/src/metabase/view_log/events/view_log.clj b/src/metabase/view_log/events/view_log.cljindex 22b05ad5a25..f0717239bb6 100644--- a/src/metabase/view_log/events/view_log.clj+++ b/src/metabase/view_log/events/view_log.clj@@ -14,6 +14,7 @@[metabase.util.log :as log][metabase.util.malli :as mu][metabase.view-log.db :as view-log.db]+ [metabase.view-log.schema :as view-log.schema][methodical.core :as m][steffan-westcott.clj-otel.api.trace.span :as span][toucan2.core :as t2]))@@ -101,7 +102,9 @@and inserted asynchronously, so they may not be visible in the view log for upto [[record-view-interval-seconds]] (plus they are lost on non-graceful shutdown); we consider that an acceptabletrade for not paying for a synchronous INSERT on every read/query request."- [view-or-views :- [:or :map [:sequential :map]]]+ [view-or-views :- [:or+ ::view-log.schema/view-log.update+ [:sequential ::view-log.schema/view-log.update]]](span/with-span!{:name "record-view!"}(when (premium-features/log-enabled?)diff --git a/src/metabase/view_log/schema.clj b/src/metabase/view_log/schema.cljindex 25779d59a45..72601de4ae4 100644--- a/src/metabase/view_log/schema.clj+++ b/src/metabase/view_log/schema.clj@@ -7,31 +7,14 @@(mr/def ::view-log.metadata"The `:metadata` column of a ViewLog, decoded."- :map)+ [:map {:closed true}])(mr/def ::view-log"A ViewLog as selected from the app DB: every column of `:view_log`."- [:map {:closed true}- [:id ms/PositiveInt]- [:user_id [:maybe ::lib.schema.id/user]]- [:model [:or :keyword :string]]- [:model_id [:maybe :int]]- [:timestamp ms/TemporalInstant]- [:metadata [:maybe ::view-log.metadata]]- [:has_access [:maybe :boolean]]- [:context [:maybe [:or :keyword :string]]]- [:embedding_client [:maybe :string]]- [:embedding_sdk_version [:maybe :string]]- [:auth_method [:maybe [:or :keyword :string]]]- [:tenant_id [:maybe ms/PositiveInt]]- [:embedding_hostname [:maybe :string]]- [:embedding_path [:maybe :string]]- [:user_agent [:maybe :string]]- [:ip_address [:maybe :string]]- [:sanitized_user_agent [:maybe :string]]- [:embedding_route [:maybe :string]]- [:metabase_version [:maybe :string]]- [:embedding_client_identifier [:maybe :string]]])+ [:merge+ ::view-log.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::view-log.update"What an update (or insert) of a ViewLog accepts: every column of `:view_log` except `id`, all optional."diff --git a/src/metabase/warehouse_schema/metadata_from_qp.clj b/src/metabase/warehouse_schema/metadata_from_qp.cljindex e6f8891b109..b6d5ec5328c 100644--- a/src/metabase/warehouse_schema/metadata_from_qp.clj+++ b/src/metabase/warehouse_schema/metadata_from_qp.clj@@ -11,17 +11,24 @@[metabase.query-processor.interface :as qp.i][metabase.query-processor.schema :as qp.schema][metabase.util.malli :as mu]- [metabase.util.malli.schema :as ms][metabase.warehouse-schema.db :as warehouse-schema.db][metabase.warehouse-schema.metadata-queries :as schema.metadata-queries]+ [metabase.warehouse-schema.schema :as warehouse-schema.schema][toucan2.core :as t2]))+(def ^:private FieldOrColumn+ "A Field, as an app DB row or as Lib metadata."+ [:multi {:dispatch (fn [x] (if (:lib/type x) :lib-metadata :row))}+ [:lib-metadata ::lib.schema.metadata/column]+ [:row ::warehouse-schema.schema/field]])+(mu/defn table-query"Runs the `mbql-query` where the source table is `table-id` and returns the result.Add the required filters if the table requires it,see [[metabase.warehouse-schema.metadata-queries/add-required-filters-if-needed]] for more details. Also takes anoptional `rff`, use the default rff if not provided."- ([table-id query-xform]+ ([table-id :- ::lib.schema.id/table+ query-xform :- [:=> [:cat ::lib.schema/query] ::lib.schema/query]](table-query table-id query-xform nil))([table-id :- ::lib.schema.id/table@@ -39,12 +46,10 @@(mu/defn field-distinct-count"Return the distinct count of `field`."- ([field]+ ([field :- FieldOrColumn](field-distinct-count field nil))- ([field :- [:or- ::lib.schema.metadata/column- (ms/InstanceOf :model/Field)]+ ([field :- FieldOrColumnlimit :- [:maybe pos-int?]](let [field (cond-> field(t2/model field) (lib-be/instance->metadata :metadata/column))]@@ -58,9 +63,7 @@(mu/defn field-count"Return the count of `field`."- [field :- [:or- ::lib.schema.metadata/column- (ms/InstanceOf :model/Field)]]+ [field :- FieldOrColumn](let [field (cond-> field(t2/model field) (lib-be/instance->metadata :metadata/column))](-> (table-query (:table-id field) #(lib/aggregate % (lib/count field)))diff --git a/src/metabase/warehouse_schema/models/dimension.clj b/src/metabase/warehouse_schema/models/dimension.cljindex e2bc4a3b81a..1b4939fd350 100644--- a/src/metabase/warehouse_schema/models/dimension.clj+++ b/src/metabase/warehouse_schema/models/dimension.clj@@ -5,6 +5,7 @@(:require[metabase.models.interface :as mi][metabase.models.serialization :as serdes]+ [metabase.warehouse-schema.schema][methodical.core :as methodical][toucan2.core :as t2]))diff --git a/src/metabase/warehouse_schema/models/field.clj b/src/metabase/warehouse_schema/models/field.cljindex a1b9f5c74d6..81c5cb6f1dd 100644--- a/src/metabase/warehouse_schema/models/field.clj+++ b/src/metabase/warehouse_schema/models/field.clj@@ -19,6 +19,7 @@[metabase.warehouse-schema.db :as warehouse-schema.db][metabase.warehouse-schema.humanization :as humanization][metabase.warehouse-schema.models.field-values :as field-values]+ [metabase.warehouse-schema.schema][metabase.warehouses.models.database :as database][methodical.core :as methodical][potemkin :as p]diff --git a/src/metabase/warehouse_schema/models/field_user_settings.clj b/src/metabase/warehouse_schema/models/field_user_settings.cljindex 60019d8fd6c..459c1421ed1 100644--- a/src/metabase/warehouse_schema/models/field_user_settings.clj+++ b/src/metabase/warehouse_schema/models/field_user_settings.clj@@ -68,7 +68,7 @@(mu/defn upsert-user-settings"Record the user-settable Field columns present in `settings` as the user values of `field`."- [{:keys [id]} :- [:map [:id ::lib.schema.id/field]]+ [{:keys [id]} :- ::warehouse-schema.schema/fieldsettings :- ::warehouse-schema.schema/field.update](let [settings (u/select-keys-when settings :present field/field-user-settings)](when (seq settings)@@ -89,7 +89,7 @@(mu/defn unset-user-settings!"Drop the user values of the Field columns `ks` for `field`."- [{:keys [id]} :- [:map [:id ::lib.schema.id/field]]+ [{:keys [id]} :- ::warehouse-schema.schema/fieldks :- [:sequential (into [:enum] warehouse-schema-overlay/user-settable-field-columns)]](when (warehouse-schema.db/field-user-settings-exist? id)(warehouse-schema.db/update-field-user-settings!diff --git a/src/metabase/warehouse_schema/models/field_values.clj b/src/metabase/warehouse_schema/models/field_values.cljindex 1e61496b200..1d99341fd2b 100644--- a/src/metabase/warehouse_schema/models/field_values.clj+++ b/src/metabase/warehouse_schema/models/field_values.clj@@ -43,6 +43,7 @@[metabase.util.malli.schema :as ms][metabase.warehouse-schema.db :as warehouse-schema.db][metabase.warehouse-schema.metadata-from-qp :as metadata-from-qp]+ [metabase.warehouse-schema.schema][methodical.core :as methodical][toucan2.core :as t2]))diff --git a/src/metabase/warehouse_schema/models/table.clj b/src/metabase/warehouse_schema/models/table.cljindex 1390222863a..68c4460a0e1 100644--- a/src/metabase/warehouse_schema/models/table.clj+++ b/src/metabase/warehouse_schema/models/table.clj@@ -17,6 +17,7 @@[metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay][metabase.warehouse-schema.db :as warehouse-schema.db][metabase.warehouse-schema.humanization :as humanization]+ [metabase.warehouse-schema.schema][methodical.core :as methodical][toucan2.core :as t2]))diff --git a/src/metabase/warehouse_schema/models/table_user_settings.clj b/src/metabase/warehouse_schema/models/table_user_settings.cljindex 5ff16161f53..9d65af5ebe4 100644--- a/src/metabase/warehouse_schema/models/table_user_settings.clj+++ b/src/metabase/warehouse_schema/models/table_user_settings.clj@@ -100,7 +100,7 @@(mu/defn upsert-user-settings"Record the user-settable Table columns present in `settings` as the user values of `table`."- [{:keys [id]} :- [:map [:id ::lib.schema.id/table]]+ [{:keys [id]} :- ::warehouse-schema.schema/tablesettings :- ::warehouse-schema.schema/table.update](let [settings (u/select-keys-when settings :present warehouse-schema-overlay/user-settable-table-columns)](when (seq settings)@@ -138,7 +138,7 @@(mu/defn unset-user-settings!"Drop the user values of the Table columns `ks` for `table`."- [{:keys [id]} :- [:map [:id ::lib.schema.id/table]]+ [{:keys [id]} :- ::warehouse-schema.schema/tableks :- [:sequential (into [:enum] warehouse-schema-overlay/user-settable-table-columns)]](when (warehouse-schema.db/table-user-settings-exist? id)(warehouse-schema.db/update-table-user-settings!diff --git a/src/metabase/warehouse_schema/schema.clj b/src/metabase/warehouse_schema/schema.cljindex a036a249fdf..58c15abab28 100644--- a/src/metabase/warehouse_schema/schema.clj+++ b/src/metabase/warehouse_schema/schema.clj@@ -3,19 +3,16 @@[metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.metadata.fingerprint :as lib.schema.metadata.fingerprint][metabase.util.malli.registry :as mr]- [metabase.util.malli.schema :as ms]))+ [metabase.util.malli.schema :as ms]+ [metabase.warehouses.schema]))(mr/def ::dimension"A Dimension as selected from the app DB: every column of `:dimension`."- [:map {:closed true}- [:id ms/PositiveInt]- [:field_id ::lib.schema.id/field]- [:name :string]- [:type [:or :keyword :string]]- [:human_readable_field_id [:maybe ::lib.schema.id/field]]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]- [:entity_id :string]])+ [:merge+ ::dimension.update+ [:map {:closed true}+ [:id ms/PositiveInt]+ [:human_readable_field {:optional true} [:maybe [:ref ::field]]]]])(mr/def ::dimension.update"What an update (or insert) of a Dimension accepts: every column of `:dimension` except `id`, all optional."@@ -30,52 +27,18 @@(mr/def ::field.settings"The `:settings` column of a Field, decoded."- :map)+ ms/VisualizationSettings)(mr/def ::field"A Field as selected from the app DB: every column of `:metabase_field`."- [:map {:closed true}- [:id ::lib.schema.id/field]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]- [:name :string]- [:base_type [:or :keyword :string]]- [:semantic_type [:maybe [:or :keyword :string]]]- [:active :boolean]- [:description [:maybe :string]]- [:preview_display :boolean]- [:position :int]- [:table_id ::lib.schema.id/table]- [:parent_id [:maybe ms/PositiveInt]]- [:display_name [:maybe :string]]- [:visibility_type [:or :keyword :string]]- [:fk_target_field_id [:maybe ::lib.schema.id/field]]- [:last_analyzed [:maybe ms/TemporalInstant]]- [:points_of_interest [:maybe :string]]- [:caveats [:maybe :string]]- [:fingerprint [:maybe ::lib.schema.metadata.fingerprint/fingerprint]]- [:fingerprint_version :int]- [:database_type [:or :keyword :string]]- [:has_field_values [:maybe [:or :keyword :string]]]- [:settings [:maybe ::field.settings]]- [:database_position :int]- [:custom_position :int]- [:effective_type [:maybe [:or :keyword :string]]]- [:coercion_strategy [:maybe [:or :keyword :string]]]- [:nfc_path [:maybe [:sequential [:or :string :keyword]]]]- [:database_required :boolean]- [:json_unfolding :boolean]- [:database_is_auto_increment :boolean]- [:database_indexed [:maybe :boolean]]- [:database_partitioned [:maybe :boolean]]- [:is_defective_duplicate {:optional true} :boolean]- [:unique_field_helper {:optional true} [:maybe :int]]- [:database_is_pk [:maybe :boolean]]- [:database_is_nullable [:maybe :boolean]]- [:database_is_generated [:maybe :boolean]]- [:database_default [:maybe :string]]- [:dimension_interestingness [:maybe number?]]- [:data_sensitivity [:maybe [:or :keyword :string]]]])+ [:merge+ ::field.update+ [:map {:closed true}+ [:id ::lib.schema.id/field]+ [:unique_field_helper {:optional true} [:maybe :int]]+ [:dimensions {:optional true} [:maybe [:sequential [:ref ::dimension]]]]+ [:name_field {:optional true} [:maybe :metabase.queries.schema/param-field.name-field]]+ [:target {:optional true} [:maybe [:ref ::field]]]]])(mr/def ::field.update"What an update (or insert) of a Field accepts: every column of `:metabase_field` except `id`, all optional."@@ -122,32 +85,13 @@(mr/def ::field-user-settings.settings"The `:settings` column of a FieldUserSettings, decoded."- :map)+ ms/VisualizationSettings)(mr/def ::field-user-settings"A FieldUserSettings as selected from the app DB: every column of `:metabase_field_user_settings`."- [:map {:closed true}- [:field_id ::lib.schema.id/field]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]- [:semantic_type [:maybe [:or :keyword :string]]]- [:description [:maybe :string]]- [:display_name [:maybe :string]]- [:visibility_type [:maybe [:or :keyword :string]]]- [:fk_target_field_id [:maybe ::lib.schema.id/field]]- [:has_field_values [:maybe [:or :keyword :string]]]- [:effective_type [:maybe [:or :keyword :string]]]- [:coercion_strategy [:maybe [:or :keyword :string]]]- [:caveats [:maybe :string]]- [:points_of_interest [:maybe :string]]- [:nfc_path [:maybe [:sequential [:or :string :keyword]]]]- [:json_unfolding [:maybe :boolean]]- [:settings [:maybe ::field-user-settings.settings]]- [:data_sensitivity [:maybe [:or :keyword :string]]]- [:custom_position [:maybe :int]]- [:description_set :boolean]- [:semantic_type_set :boolean]- [:fk_target_field_id_set :boolean]])+ [:merge+ ::field-user-settings.update+ [:map {:closed true}]])(mr/def ::field-user-settings.update"What an update (or insert) of a FieldUserSettings accepts: every column of `:metabase_field_user_settings` except `id`, all optional."@@ -176,17 +120,10 @@(mr/def ::field-values"A FieldValues as selected from the app DB: every column of `:metabase_fieldvalues`."- [:map {:closed true}- [:id ms/PositiveInt]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]- [:values [:maybe ms/FieldValues]]- [:human_readable_values [:maybe ms/FieldValues]]- [:field_id ::lib.schema.id/field]- [:has_more_values [:maybe :boolean]]- [:type [:or :keyword :string]]- [:hash_key [:maybe :string]]- [:last_used_at ms/TemporalInstant]])+ [:merge+ ::field-values.update+ [:map {:closed true}+ [:id ms/PositiveInt]]])(mr/def ::field-values.update"What an update (or insert) of a FieldValues accepts: every column of `:metabase_fieldvalues` except `id`, all optional."@@ -203,31 +140,9 @@(mr/def ::table-user-settings"A TableUserSettings as selected from the app DB: every column of `:metabase_table_user_settings`."- [:map {:closed true}- [:table_id ::lib.schema.id/table]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]- [:display_name [:maybe :string]]- [:description [:maybe :string]]- [:entity_type [:maybe [:or :keyword :string]]]- [:visibility_type [:maybe [:or :keyword :string]]]- [:caveats [:maybe :string]]- [:points_of_interest [:maybe :string]]- [:data_layer [:maybe [:or :keyword :string]]]- [:data_source [:maybe [:or :keyword :string]]]- [:owner_email [:maybe :string]]- [:owner_user_id [:maybe ::lib.schema.id/user]]- [:field_order [:maybe [:or :keyword :string]]]- [:show_in_getting_started [:maybe :boolean]]- [:data_authority [:maybe [:or :keyword :string]]]- [:is_published [:maybe :boolean]]- [:collection_id [:maybe ::lib.schema.id/collection]]- [:description_set :boolean]- [:visibility_type_set :boolean]- [:caveats_set :boolean]- [:points_of_interest_set :boolean]- [:data_layer_set :boolean]- [:data_source_set :boolean]])+ [:merge+ ::table-user-settings.update+ [:map {:closed true}]])(mr/def ::table-user-settings.update"What an update (or insert) of a TableUserSettings accepts: every column of@@ -260,41 +175,20 @@(mr/def ::table"A Table as selected from the app DB: every column of `:metabase_table`."- [:map {:closed true}- [:id ::lib.schema.id/table]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]- [:name :string]- [:description [:maybe :string]]- [:entity_type [:maybe [:or :keyword :string]]]- [:active :boolean]- [:db_id ::lib.schema.id/database]- [:display_name [:maybe :string]]- [:visibility_type [:maybe [:or :keyword :string]]]- [:schema [:maybe :string]]- [:points_of_interest [:maybe :string]]- [:caveats [:maybe :string]]- [:show_in_getting_started :boolean]- [:field_order [:or :keyword :string]]- [:initial_sync_status [:or :keyword :string]]- [:is_upload :boolean]- [:database_require_filter [:maybe :boolean]]- [:estimated_row_count [:maybe :int]]- [:view_count :int]- [:is_defective_duplicate {:optional true} :boolean]- [:unique_table_helper {:optional true} [:maybe :string]]- [:deactivated_at [:maybe ms/TemporalInstant]]- [:archived_at [:maybe ms/TemporalInstant]]- [:is_writable [:maybe :boolean]]- [:data_authority [:or :keyword :string]]- [:data_source [:maybe [:or :keyword :string]]]- [:data_layer [:maybe [:or :keyword :string]]]- [:owner_email [:maybe :string]]- [:owner_user_id [:maybe ::lib.schema.id/user]]- [:collection_id [:maybe ::lib.schema.id/collection]]- [:is_published :boolean]- [:transform_id [:maybe ::lib.schema.id/transform]]- [:transform_target :boolean]])+ [:merge+ ::table.update+ [:map {:closed true}+ [:id ::lib.schema.id/table]+ [:unique_table_helper {:optional true} [:maybe :string]]+ [:db {:optional true} [:maybe [:ref :metabase.warehouses.schema/database]]]+ [:fields {:optional true} [:maybe [:sequential ::field]]]+ [:transform {:optional true} [:maybe [:ref :metabase.transforms.schema/transform]]]+ [:owner {:optional true} [:maybe [:map {:closed true}+ [:id {:optional true} ::lib.schema.id/user]+ [:email {:optional true} :string]+ [:first_name {:optional true} [:maybe :string]]+ [:last_name {:optional true} [:maybe :string]]+ [:common_name {:optional true} [:maybe :string]]]]]]])(mr/def ::table.update"What an update (or insert) of a Table accepts: every column of `:metabase_table` except `id`, all optional."diff --git a/src/metabase/warehouse_schema_overlay/core.clj b/src/metabase/warehouse_schema_overlay/core.cljindex 50eec856971..c6f7fc270f6 100644--- a/src/metabase/warehouse_schema_overlay/core.clj+++ b/src/metabase/warehouse_schema_overlay/core.clj@@ -72,7 +72,7 @@([{:keys [alias user-settings?]:or {alias (t2/table-name :model/Field)- user-settings? true}} :- [:maybe [:map+ user-settings? true}} :- [:maybe [:map {:closed true}[:alias {:optional true} :keyword][:user-settings? {:optional true} :boolean]]]][(if user-settings?@@ -153,7 +153,7 @@([{:keys [alias user-settings?]:or {alias (t2/table-name :model/Table)- user-settings? true}} :- [:maybe [:map+ user-settings? true}} :- [:maybe [:map {:closed true}[:alias {:optional true} :keyword][:user-settings? {:optional true} :boolean]]]][(if user-settings?diff --git a/src/metabase/warehouse_schema_rest/api/table.clj b/src/metabase/warehouse_schema_rest/api/table.cljindex bdca117e839..e004514e73e 100644--- a/src/metabase/warehouse_schema_rest/api/table.clj+++ b/src/metabase/warehouse_schema_rest/api/table.clj@@ -11,7 +11,6 @@[metabase.events.core :as events][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]- [metabase.lib.schema.id :as lib.schema.id][metabase.models.interface :as mi][metabase.permissions.core :as perms][metabase.premium-features.core :as premium-features]@@ -177,11 +176,46 @@(fn [response](dissoc response :json_query :context :cached :average_execution_time))))))))))+(def ^:private TableUpdateBodySingle+ "Body of `PUT /api/table/:id`."+ [:map {:closed true}+ [:display_name {:optional true} [:maybe ms/NonBlankString]]+ [:entity_type {:optional true} [:maybe EntityType]]+ [:visibility_type {:optional true} [:maybe TableVisibilityType]]+ [:description {:optional true} [:maybe :string]]+ [:caveats {:optional true} [:maybe :string]]+ [:points_of_interest {:optional true} [:maybe :string]]+ [:show_in_getting_started {:optional true} [:maybe :boolean]]+ [:field_order {:optional true} [:maybe FieldOrder]]+ [:data_authority {:optional true} [:maybe ::data-authority-write]]+ [:data_source {:optional true} [:maybe :string]]+ [:data_layer {:optional true} [:maybe :string]]+ [:owner_email {:optional true} [:maybe :string]]+ [:owner_user_id {:optional true} [:maybe :int]]+ [:collection_id {:optional true} [:maybe ms/PositiveInt]]])++(def ^:private TableUpdateBodyBulk+ "Body of the deprecated `PUT /api/table/`."+ [:map {:closed true}+ [:ids [:sequential ms/PositiveInt]]+ [:display_name {:optional true} [:maybe ms/NonBlankString]]+ [:entity_type {:optional true} [:maybe EntityType]]+ [:visibility_type {:optional true} [:maybe TableVisibilityType]]+ [:description {:optional true} [:maybe :string]]+ [:caveats {:optional true} [:maybe :string]]+ [:points_of_interest {:optional true} [:maybe :string]]+ [:show_in_getting_started {:optional true} [:maybe :boolean]]+ [:data_authority {:optional true} [:maybe ::data-authority-write]]+ [:data_source {:optional true} [:maybe :string]]+ [:data_layer {:optional true} [:maybe :string]]+ [:owner_email {:optional true} [:maybe :string]]+ [:owner_user_id {:optional true} [:maybe :int]]])+(mu/defn ^:private update-table!*"Takes an existing table and the changes, updates in the database and optionally calls `table/update-field-positions!`if field positions have changed."- [{:keys [id] :as existing-table} :- [:map [:id ::lib.schema.id/table]]- body]+ [{:keys [id] :as existing-table} :- :metabase.warehouse-schema.schema/table+ body :- [:or TableUpdateBodySingle TableUpdateBodyBulk]](when-let [changes (-> body(u/select-keys-when:non-nil [:display_name :show_in_getting_started :entity_type :field_order :collection_id]@@ -253,21 +287,7 @@[{:keys [id]} :- [:map {:closed true}[:id ms/PositiveInt]]_query-params- body :- [:map {:closed true}- [:display_name {:optional true} [:maybe ms/NonBlankString]]- [:entity_type {:optional true} [:maybe EntityType]]- [:visibility_type {:optional true} [:maybe TableVisibilityType]]- [:description {:optional true} [:maybe :string]]- [:caveats {:optional true} [:maybe :string]]- [:points_of_interest {:optional true} [:maybe :string]]- [:show_in_getting_started {:optional true} [:maybe :boolean]]- [:field_order {:optional true} [:maybe FieldOrder]]- [:data_authority {:optional true} [:maybe ::data-authority-write]]- [:data_source {:optional true} [:maybe :string]]- [:data_layer {:optional true} [:maybe :string]]- [:owner_email {:optional true} [:maybe :string]]- [:owner_user_id {:optional true} [:maybe :int]]- [:collection_id {:optional true} [:maybe ms/PositiveInt]]]]+ body :- TableUpdateBodySingle](first (update-tables! [id] body)));; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to@@ -280,20 +300,7 @@Deprecated, should use PUT /table/edit from now on."[_route-params_query-params- {:keys [ids], :as body} :- [:map {:closed true}- [:ids [:sequential ms/PositiveInt]]- [:display_name {:optional true} [:maybe ms/NonBlankString]]- [:entity_type {:optional true} [:maybe EntityType]]- [:visibility_type {:optional true} [:maybe TableVisibilityType]]- [:description {:optional true} [:maybe :string]]- [:caveats {:optional true} [:maybe :string]]- [:points_of_interest {:optional true} [:maybe :string]]- [:show_in_getting_started {:optional true} [:maybe :boolean]]- [:data_authority {:optional true} [:maybe ::data-authority-write]]- [:data_source {:optional true} [:maybe :string]]- [:data_layer {:optional true} [:maybe :string]]- [:owner_email {:optional true} [:maybe :string]]- [:owner_user_id {:optional true} [:maybe :int]]]]+ {:keys [ids], :as body} :- TableUpdateBodyBulk](update-tables! ids body));; TODO (Cam 10/28/25) -- fix this endpoint so it uses kebab-case for query parameters for consistency with the rest@@ -444,7 +451,7 @@(mu/defn- update-csv!"This helper function exists to make testing the POST /api/table/:id/{action}-csv endpoints easier."- [options :- [:map+ [options :- [:map {:closed true}[:table-id ms/PositiveInt][:filename :string][:file (ms/InstanceOfClass java.io.File)]diff --git a/src/metabase/warehouses/models/database.clj b/src/metabase/warehouses/models/database.cljindex 83d4c463837..4b65bf3a9ed 100644--- a/src/metabase/warehouses/models/database.clj+++ b/src/metabase/warehouses/models/database.clj@@ -30,6 +30,7 @@[metabase.util.quick-task :as quick-task][metabase.warehouses.db :as warehouses.db][metabase.warehouses.provider-detection :as provider-detection]+ [metabase.warehouses.schema][metabase.warehouses.settings :as warehouses.settings][methodical.core :as methodical][toucan2.core :as t2]@@ -405,6 +406,7 @@(cond-> database;; TODO - this is only really needed for API responses. This should be a `hydrate` thing instead!(and driver+ (:id database)(driver.impl/registered? driver))(assoc :features (driver.u/features driver (t2.realize/realize database)))@@ -631,9 +633,11 @@driver can't be clearly determined, this simply returns the default set (driver.u/default-sensitive-fields)."[database](if (and (some? database) (not-empty database))- (let [driver (driver.u/database->driver database)]+ (let [driver (if-let [engine (:engine database)]+ (keyword engine)+ (driver.u/database->driver (:id database)))](if (some? driver)- (driver.u/sensitive-fields (driver.u/database->driver database))+ (driver.u/sensitive-fields driver)driver.u/default-sensitive-fields))driver.u/default-sensitive-fields))diff --git a/src/metabase/warehouses/schema.clj b/src/metabase/warehouses/schema.cljindex 03985a41e4e..3de3b7781dc 100644--- a/src/metabase/warehouses/schema.clj+++ b/src/metabase/warehouses/schema.clj@@ -2,65 +2,57 @@"Malli schemas for the warehouses module."(:require[metabase.lib.schema.id :as lib.schema.id]+ [metabase.util.cron][metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms]))+(mr/def ::database.undecryptable-column+ "An encrypted JSON column of a Database read without the key that encrypted it, which the model's transform logs and+ returns as the raw ciphertext string."+ :string)+(mr/def ::database.details"The `:details` column of a Database, decoded."- :map)+ [:or ms/DatabaseDetails ::database.undecryptable-column])(mr/def ::database.settings"The `:settings` column of a Database, decoded."- :map)+ [:or ms/DatabaseSettings ::database.undecryptable-column])(mr/def ::database.dbms-version"The `:dbms_version` column of a Database, decoded."- :map)+ [:map {:closed true}+ [:flavor {:optional true} :string]+ [:version {:optional true} :string]+ [:semantic-version {:optional true} [:or+ [:sequential :int]+ [:map {:closed true} [:major :int] [:minor :int]]]]+ [:cloud {:optional true} :boolean]])(mr/def ::database.write-data-details"The `:write_data_details` column of a Database, decoded."- :map)+ [:or ms/DatabaseDetails ::database.undecryptable-column])(mr/def ::database.admin-details"The `:admin_details` column of a Database, decoded."- :map)+ [:or ms/DatabaseDetails ::database.undecryptable-column])(mr/def ::database"A Database as selected from the app DB: every column of `:metabase_database`, plus `:features` added by the model's after-select hook."- [:map {:closed true}- [:id ::lib.schema.id/database]- [:created_at ms/TemporalInstant]- [:updated_at ms/TemporalInstant]- [:name :string]- [:description [:maybe :string]]- [:details ::database.details]- [:engine [:or :keyword :string]]- [:is_sample :boolean]- [:is_full_sync :boolean]- [:points_of_interest [:maybe :string]]- [:caveats [:maybe :string]]- [:metadata_sync_schedule :string]- [:cache_field_values_schedule [:maybe :string]]- [:timezone [:maybe :string]]- [:is_on_demand :boolean]- [:auto_run_queries :boolean]- [:refingerprint [:maybe :boolean]]- [:cache_ttl [:maybe :int]]- [:initial_sync_status [:or :keyword :string]]- [:creator_id [:maybe ::lib.schema.id/user]]- [:settings [:maybe ::database.settings]]- [:dbms_version [:maybe ::database.dbms-version]]- [:is_audit :boolean]- [:uploads_enabled :boolean]- [:uploads_schema_name [:maybe :string]]- [:uploads_table_prefix [:maybe :string]]- [:is_attached_dwh :boolean]- [:router_database_id [:maybe ::lib.schema.id/database]]- [:provider_name [:maybe :string]]- [:write_data_details [:maybe ::database.write-data-details]]- [:admin_details [:maybe ::database.admin-details]]- [:is_stub :boolean]- [:features {:optional true} [:maybe [:set :keyword]]]])+ [:merge+ ::database.update+ [:map {:closed true}+ [:id ::lib.schema.id/database]+ [:features {:optional true} [:maybe [:set :keyword]]]+ [:can-manage {:optional true} [:maybe :boolean]]+ [:can_upload {:optional true} [:maybe :boolean]]+ [:tables {:optional true} [:maybe [:sequential [:ref :metabase.warehouse-schema.schema/table]]]]+ [:native_permissions {:optional true} [:maybe [:enum :write :none]]]+ [:router_user_attribute {:optional true} [:maybe :string]]+ [:schedules {:optional true} [:maybe [:map {:closed true}+ [:metadata_sync :metabase.util.cron/ScheduleMap]+ [:cache_field_values [:maybe :metabase.util.cron/ScheduleMap]]]]]+ [:transforms_permissions {:optional true} [:maybe [:enum :write :none]]]]])(mr/def ::database.update"What an update (or insert) of a Database accepts: every column of `:metabase_database` except `id`, all optional."@@ -96,3 +88,9 @@[:write_data_details {:optional true} [:maybe ::database.write-data-details]][:admin_details {:optional true} [:maybe ::database.admin-details]][:is_stub {:optional true} [:maybe :boolean]]])++(mr/def ::database-or-metadata+ "A Database as an app DB row or as Lib metadata."+ [:multi {:dispatch (fn [x] (if (:lib/type x) :lib-metadata :row))}+ [:lib-metadata :metabase.lib.schema.metadata/database]+ [:row ::database]])diff --git a/src/metabase/warehouses/util.clj b/src/metabase/warehouses/util.cljindex 78541d0d7bc..fad77d6b595 100644--- a/src/metabase/warehouses/util.clj+++ b/src/metabase/warehouses/util.clj@@ -12,12 +12,12 @@(mu/defn get-database"Retrieve database respecting `include-editable-data-model?`, `exclude-uneditable-details?` and `include-mirror-databases?`"- ([id] (get-database id {}))+ ([id :- ms/PositiveInt] (get-database id {}))([id :- ms/PositiveInt{:keys [include-editable-data-model?exclude-uneditable-details?include-destination-databases?]}- :- [:map+ :- [:map {:closed true}[:include-editable-data-model? {:optional true :default false} ms/MaybeBooleanValue][:exclude-uneditable-details? {:optional true :default false} ms/MaybeBooleanValue][:include-destination-databases? {:optional true :default false} ms/MaybeBooleanValue]]]@@ -82,7 +82,7 @@the details used to successfully connect. Otherwise returns a map with the connection error message. (This map willalso contain the key `:valid` = `false`, which you can use to distinguish an error from valid details.)"[engine :- [:or keyword? string?]- details :- :map]+ details :- ms/DatabaseDetails](let [;; Try SSL first if SSL is supported and not already enabled;; If not successful or not applicable, details-with-ssl will be nildetails-with-ssl (assoc details :ssl true)diff --git a/src/metabase/warehouses_rest/api.clj b/src/metabase/warehouses_rest/api.cljindex cbad4598b12..703abb47885 100644--- a/src/metabase/warehouses_rest/api.clj+++ b/src/metabase/warehouses_rest/api.clj@@ -15,7 +15,6 @@[metabase.driver.util :as driver.u][metabase.events.core :as events][metabase.lib-be.core :as lib-be]- [metabase.lib-be.schema :as lib-be.schema][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.lib.schema.id :as lib.schema.id]@@ -46,6 +45,7 @@[metabase.warehouses-rest.db :as warehouses-rest.db][metabase.warehouses.core :as warehouses][metabase.warehouses.models.database :as database]+ [metabase.warehouses.schema :as warehouses.schema][toucan2.core :as t2]))(set! *warn-on-reflection* true)@@ -114,7 +114,7 @@permissions; there was a specific option where you could give a Perms Group permissions to run existing Cards withnative queries, but not to create new ones. With the advent of what is currently being called 'Space-AgePermissions', all Cards' permissions are based on their parent Collection, removing the need for native read perms."- [dbs :- [:maybe [:sequential :map]]]+ [dbs :- [:maybe [:sequential ::warehouses.schema/database]]](for [db dbs](assoc db:native_permissions@@ -132,7 +132,7 @@[:transforms_permissions [:enum :write :none]]]]]"For each database in DBS add a `:transforms_permissions` field describing the current user's permissions forcreating/running Transforms. Will be either `:write` or `:none`."- [dbs :- [:maybe [:sequential :map]]]+ [dbs :- [:maybe [:sequential ::warehouses.schema/database]]](for [db dbs](assoc db:transforms_permissions@@ -174,8 +174,7 @@"Since cumulative count and cumulative sum aggregations are done in Clojure-land we can't use Cards that use querieswith those aggregations as source queries. This function determines whether `card` is using one of those queries sowe can filter it out in Clojure-land."- [{query :dataset_query, :as _card} :- [:map- [:dataset_query ::lib-be.schema/maybe-legacy-or-empty-query]]]+ [{query :dataset_query, :as _card} :- ::queries.schema/card](match/match-one (lib/aggregations query) [#{:cum-count :cum-sum} & _] true))(defn card-can-be-used-as-source-query?@@ -187,10 +186,10 @@(defn- ids-of-dbs-that-support-source-queries [];; the nested-queries check only reads the engine — don't realize full rows (decrypted :details etc.) for it- (set (keep (fn [db]+ (set (keep (fn [{:keys [id engine]}](try- (when (driver.u/supports? (driver.u/database->driver db) :nested-queries db)- (:id db))+ (when (driver.u/supports? (keyword engine) :nested-queries nil)+ id)(catch Throwable e(log/errorf "Error determining whether Database supports nested queries: %s" (ex-message e)))))(warehouses-rest.db/database-engines))))@@ -198,7 +197,11 @@(mu/defn- source-query-cards"Fetch the Cards that can be used as source queries (e.g. presented as virtual tables)."[card-type :- ::queries.schema/card.type- & {:keys [collection-scope xform], :or {xform identity}}]+ & {:keys [collection-scope xform], :or {xform identity}}+ :- [:maybe [:map {:closed true}+ [:collection-scope {:optional true}+ [:maybe [:or [:= :root] [:set ::lib.schema.id/collection] [:sequential ::lib.schema.id/collection]]]]+ [:xform {:optional true} [:maybe [:fn ifn?]]]]]](when-let [ids-of-dbs-that-support-source-queries (not-empty (ids-of-dbs-that-support-source-queries))](transduce(comp (map (partial mi/do-after-select :model/Card))@@ -219,13 +222,17 @@(This takes the Cards from `source-query-cards` and returns them in a format suitable for consumption by the QueryBuilder.)"[card-type :- ::queries.schema/card.type- & {:keys [include-fields?]}]+ & {:keys [include-fields?]}+ :- [:maybe [:map {:closed true} [:include-fields? {:optional true} [:maybe :boolean]]]]](schema.table/cards->virtual-tables (source-query-cards card-type):include-fields? include-fields?))(mu/defn- saved-cards-virtual-db-metadata[card-type :- ::queries.schema/card.type- & {:keys [include-tables? include-fields?]}]+ & {:keys [include-tables? include-fields?]}+ :- [:maybe [:map {:closed true}+ [:include-tables? {:optional true} [:maybe :boolean]]+ [:include-fields? {:optional true} [:maybe :boolean]]]]](when (lib-be/enable-nested-queries)(cond-> {:name (trs "Saved Questions"):id lib.schema.id/saved-questions-virtual-database-id@@ -436,10 +443,10 @@(= include "tables.fields") apply-sandbox-column-filter))))))(mu/defn- check-database-exists- ([id] (check-database-exists id {}))+ ([id :- ms/PositiveInt] (check-database-exists id {}))([id :- ms/PositiveInt{:keys [include-destination-databases?]}- :- [:map+ :- [:map {:closed true}[:include-destination-databases? {:optional true :default false} ms/MaybeBooleanValue]]](api/check-404 (if (and include-destination-databases? api/*is-superuser?*)(warehouses-rest.db/database-exists? id)@@ -488,8 +495,7 @@[:include_editable_data_model {:optional true} ms/MaybeBooleanValue][:exclude_uneditable_details {:optional true} ms/MaybeBooleanValue]]](present-database- (warehouses/get-database id {:include include- :include-editable-data-model? include_editable_data_model+ (warehouses/get-database id {:include-editable-data-model? include_editable_data_model:exclude-uneditable-details? exclude_uneditable_details:include-destination-databases? true}){:include includediff --git a/src/metabase/xrays/api/automagic_dashboards.clj b/src/metabase/xrays/api/automagic_dashboards.cljindex b21b46e441b..0b2aba8dfb5 100644--- a/src/metabase/xrays/api/automagic_dashboards.clj+++ b/src/metabase/xrays/api/automagic_dashboards.clj@@ -1,6 +1,7 @@(ns metabase.xrays.api.automagic-dashboards(:require[buddy.core.codecs :as codecs]+ [clojure.walk :as walk][medley.core :as m][metabase.api.common :as api][metabase.api.macros :as api.macros]@@ -147,8 +148,9 @@[:map[:dataset_query ::ads/query]]]"Wrap query map into a Query object (mostly to facilitate type dispatch)."- [query :- :map]- (let [query (api.macros/decode-and-validate-params :body ::lib-be.schema/maybe-legacy-query query)]+ [query :- ms/RingRequestBody]+ (let [query (api.macros/decode-and-validate-params :body ::lib-be.schema/maybe-legacy-query+ (walk/keywordize-keys query))](mi/instance :model/Query(merge (queries/query->database-and-table-ids query){:dataset_query query}))))@@ -207,7 +209,9 @@(mu/defn get-automagic-dashboard"Return an automagic dashboard for entity `entity` with id `id`."- [entity :- Entity entity-id-or-query show]+ [entity :- Entity+ entity-id-or-query :- ::entity-id-or-query+ show :- [:maybe [:or [:= "all"] nat-int?]]](if (= entity :transform)(transforms.dashboard/dashboard (->entity entity entity-id-or-query))(-> (->entity entity entity-id-or-query)diff --git a/src/metabase/xrays/automagic_dashboards/combination.clj b/src/metabase/xrays/automagic_dashboards/combination.cljindex fdb7b704685..210df3c9719 100644--- a/src/metabase/xrays/automagic_dashboards/combination.clj+++ b/src/metabase/xrays/automagic_dashboards/combination.clj@@ -179,7 +179,7 @@"Generate dashcards from ground dimensions, using the base context, ground dimensions,card templates, and grounded metrics as input."[base-context :- ::ads/context- card-templates+ card-templates :- [:sequential ::ads/card-template]ground-dimensions :- ::ads/dim-name->matching-fieldsground-filters :- [:sequential ::ads/grounded-filter]grounded-metrics :- [:sequential ::ads/grounded-metric]]diff --git a/src/metabase/xrays/automagic_dashboards/comparison.clj b/src/metabase/xrays/automagic_dashboards/comparison.cljindex 8253fdddde6..0b52bb1599d 100644--- a/src/metabase/xrays/automagic_dashboards/comparison.clj+++ b/src/metabase/xrays/automagic_dashboards/comparison.clj@@ -7,9 +7,12 @@[metabase.lib.schema :as lib.schema][metabase.lib.schema.id :as lib.schema.id][metabase.models.interface :as mi]+ [metabase.queries.schema :as queries.schema][metabase.util :as u][metabase.util.i18n :refer [tru]][metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr]+ [metabase.util.malli.schema :as ms][metabase.xrays.automagic-dashboards.core :refer [->related-entity ->root automagic-analysis capitalize-first]][metabase.xrays.automagic-dashboards.filters :as filters][metabase.xrays.automagic-dashboards.names :as names]@@ -21,6 +24,16 @@(def ^:private ^{:arglists '([root])} comparison-name(comp capitalize-first (some-fn :comparison-name :full-name)))+(def ^:private ComparisonEntityInstance+ "An instance `->root` can turn into a `left`/`right` side of a comparison."+ [:or+ :metabase.warehouse-schema.schema/table+ :metabase.segments.schema/segment+ ::queries.schema/card+ :metabase.warehouse-schema.schema/field+ ::queries.schema/query+ ::ads/adhoc-question])+(mu/defn- dashboard->cards :- [:sequential ::ads/card][dashboard :- ::ads/dashboard](->> dashboard@@ -45,8 +58,39 @@:collection_id nil:id (gensym))))+(mr/def ::processed-card+ "A processed \"card\" map as it flows through this namespace: the [[clone-card]] keys, plus the render-stage+ additions [[dashboard->cards]] assocs onto it."+ [:map {:closed true}+ [:id {:optional true} [:or symbol? ::lib.schema.id/card]]+ [:dataset_query {:optional true} ::ads/query]+ [:description {:optional true} [:maybe :string]]+ [:display {:optional true} [:maybe [:or :keyword :string]]]+ [:name {:optional true} [:maybe :string]]+ [:result_metadata {:optional true} [:maybe ::queries.schema/card.result-metadata]]+ [:visualization_settings {:optional true} ms/VisualizationSettings]+ [:text {:optional true} [:maybe :string]]+ [:series {:optional true} [:maybe [:sequential [:ref ::processed-card]]]]+ [:height {:optional true} number?]+ [:position {:optional true} number?]+ [:collection_id {:optional true} [:maybe ::lib.schema.id/collection]]+ [:creator_id {:optional true} [:maybe ::lib.schema.id/user]]+ [:query_type {:optional true} [:maybe [:or :keyword :string]]]+ [:database_id {:optional true} [:maybe ::lib.schema.id/database]]+ [:table_id {:optional true} [:maybe ::lib.schema.id/table]]])++(def ^:private Card+ "See [[::processed-card]]."+ ::processed-card)++(def ^:private CardOrProcessed+ "A card as [[metabase.xrays.automagic-dashboards.populate/dashboard->cards]] renders it, or the narrower+ [[Card]]/[[::processed-card]] shape [[clone-card]] reduces it to. The functions below run on values from either+ stage of the pipeline."+ [:or ::ads/card Card])+(mu/defn- display-type :- [:maybe :keyword]- [card :- :map]+ [card :- CardOrProcessed](keyword (:display card)))(mu/defn- add-filter-clauses :- ::ads/query@@ -65,23 +109,20 @@(mu/defn- inject-filter"Inject filter clause into card."[{:keys [query-filter cell-query] :as root} :- ::ads/root- card :- [:map- [:dataset_query ::ads/query]]]+ card :- CardOrProcessed](-> card(update :dataset_query #(add-filter-clauses % (cons cell-query query-filter)))(update :series (partial map (partial inject-filter root)))))(mu/defn- multiseries?- [card :- [:map- [:dataset_query {:optional true} ::ads/query]]]+ [card :- CardOrProcessed](or (-> card :series not-empty)(when-let [query (not-empty (:dataset_query card))](or (-> query lib/aggregations count (> 1))(-> query lib/breakouts count (> 1))))))(mu/defn- overlay-comparison?- [card :- [:map- [:dataset_query {:optional true} ::ads/query]]]+ [card :- CardOrProcessed](and (-> card display-type (#{:bar :line}))(not (multiseries? card))))@@ -175,16 +216,14 @@[dashboard (max height-left height-right)]))(mu/defn- series-labels- [card :- [:map- [:dataset_query {:optional true} ::ads/query]]]+ [card :- CardOrProcessed](let [database-id (get-in card [:dataset_query :database])](get-in card [:visualization_settings :graph.series_labels](map (comp capitalize-first (partial names/metric-name database-id))(lib/aggregations (:dataset_query card))))))(mu/defn- unroll-multiseries- [card :- [:map- [:dataset_query {:optional true} ::ads/query]]]+ [card :- CardOrProcessed](if (and (multiseries? card)(-> card :display (= :line)))(for [[aggregation label] (map vector@@ -208,10 +247,9 @@(map (partial magic.util/->field segment))))(mu/defn- update-related- [related- left :- [:map- [:database ::lib.schema.id/database]]- right]+ [related :- ::ads/related+ left :- ::ads/root+ right :- ::ads/root](-> related(update :related (comp distinct conj) (-> right :entity ->related-entity))(assoc :compare (concat@@ -249,7 +287,14 @@(mu/defn comparison-dashboard"Create a comparison dashboard based on dashboard `dashboard` comparing subsets ofthe dataset defined by segments `left` and `right`."- [dashboard left right opts]+ [dashboard :- ::ads/dashboard+ left :- ComparisonEntityInstance+ right :- ComparisonEntityInstance+ opts :- [:map {:closed true}+ [:left {:optional true} [:map {:closed true}+ [:cell-query {:optional true} ::ads/root.cell-query-input]]]+ [:right {:optional true} [:map {:closed true}+ [:cell-query {:optional true} ::ads/root.cell-query-input]]]]];; disable ref validation because X-Rays does stuff in a wacko manner, it adds a bunch of filters and whatever that;; use columns from joins before adding the joins themselves (same with expressions), which is technically invalid;; at the time it happens but ends up resulting in a valid query at the end of the day. Maybe one day we can reworkdiff --git a/src/metabase/xrays/automagic_dashboards/core.clj b/src/metabase/xrays/automagic_dashboards/core.cljindex d6788dc900f..7a1544a16f5 100644--- a/src/metabase/xrays/automagic_dashboards/core.clj+++ b/src/metabase/xrays/automagic_dashboards/core.clj@@ -199,7 +199,7 @@:dashboard-templates-prefix ["table"]})(mu/defmethod ->root :model/Segment :- ::ads/root- [segment :- [:map [:definition ::segments.schema/definition]]]+ [segment :- [:map [:definition [:maybe ::segments.schema/definition]]]](let [table (->> segment :table_id xrays.db/table)]{:entity segment:full-name (tru "{0} in the {1} segment" (:display_name table) (:name segment))@@ -240,31 +240,26 @@:url (format "%sfield/%s" public-endpoint (:id field)):dashboard-templates-prefix ["field"]}))-(mu/defn- source-card-id [card-or-question :- [:map- [:dataset_query ::ads/query]]]+(mu/defn- source-card-id [card-or-question :- ::ads/card-or-question](lib/primary-source-card-id (:dataset_query card-or-question)))(mu/defn- nested-query?"Is this card or question derived from another model or question?"- [card-or-question :- [:map- [:dataset_query ::ads/query]]]+ [card-or-question :- ::ads/card-or-question](some? (source-card-id card-or-question)))(mu/defn- native-query?"Is this card or question native (SQL)?"- [{query :dataset_query, :as _card-or-question} :- [:map- [:dataset_query ::ads/query]]]+ [{query :dataset_query, :as _card-or-question} :- ::ads/card-or-question](lib/native-only-query? query))(mu/defn- source-question :- [:maybe (ms/InstanceOf :model/Card)]- [card-or-question :- [:map- [:dataset_query ::ads/query]]]+ [card-or-question :- ::ads/card-or-question](when-let [source-card-id (source-card-id card-or-question)](xrays.db/card source-card-id)))(mu/defn- table-like?- [{query :dataset_query, :as _card-or-question} :- [:map- [:dataset_query ::ads/query]]]+ [{query :dataset_query, :as _card-or-question} :- ::ads/card-or-question](and(empty? (lib/aggregations query))(empty? (lib/breakouts query))))@@ -279,8 +274,7 @@((some-fn :table-id :table_id) card-or-question))(mu/defn- source- [card :- [:map- [:dataset_query ::ads/query]]]+ [card :- ::ads/card-or-question](cond;; This is a model(= (:type card) :model) (assoc card :entity_type :entity/GenericTable)@@ -293,8 +287,7 @@:else (->> card table-id xrays.db/table)))(mu/defmethod ->root :model/Card :- ::ads/root- [card :- [:map- [:dataset_query ::ads/query]]]+ [card :- ::ads/card-or-question](let [source (source card)]{:entity card:source source@@ -441,7 +434,7 @@(update field :semantic_type keyword)(mi/instance :model/Field field)(analyze/run-classifiers field {})- (assoc field :db db)))))]+ (assoc field :db db :xrays/database-id (u/the-id db))))))](constantly source-fields))(constantly [])))))@@ -479,7 +472,7 @@[{root :root :as base-context} :- ::ads/context{template-cards :cards:keys [dashboard_filters]- :as dashboard-template}+ :as dashboard-template} :- dashboard-templates/DashboardTemplate{grounded-dimensions :dimensionsgrounded-metrics :metricsgrounded-filters :filters} :- ::ads/grounded-values]@@ -525,7 +518,7 @@(->> (m/map-vals (comp (partial map ->related-entity) u/one-or-many)))))(mu/defn- indepth- [{:keys [dashboard-templates-prefix url] :as root}+ [{:keys [dashboard-templates-prefix url] :as root} :- ::ads/root{:keys [dashboard-template-name]} :- [:maybe dashboard-templates/DashboardTemplate]](let [base-context (make-base-context root)](->> (dashboard-templates/get-dashboard-templates (concat dashboard-templates-prefix [dashboard-template-name]))@@ -561,8 +554,7 @@(hash-map :drilldown-fields))))(mu/defn- comparisons- [root :- [:map- [:database ::lib.schema.id/database]]]+ [root :- ::ads/root]{:compare (concat(for [segment (->> root :entity related/related :segments (map ->root))]{:url (str (:url root) "/compare/segment/" (-> segment :entity u/the-id))@@ -671,9 +663,9 @@(mu/defn- related"Build a balanced list of related X-rays. General composition of the list is determined for eachroot type individually via `related-selectors`. That recipe is then filled round-robin style."- [root- available-dimensions- dashboard-template :- [:maybe dashboard-templates/DashboardTemplate]]+ [root :- ::ads/root+ available-dimensions :- ::ads/dim-name->matching-fields+ dashboard-template :- [:maybe dashboard-templates/DashboardTemplate]](->> (merge (indepth root dashboard-template)(drilldown-fields root available-dimensions)(related-entities root)@@ -683,7 +675,7 @@(mu/defn- generate-dashboard"Produce a fully-populated dashboard from the base context for an item and a dashboard template."[{{:keys [show url query-filter] :as root} :root :as base-context} :- ::ads/context- {:as dashboard-template}+ {:as dashboard-template} :- dashboard-templates/DashboardTemplate{grounded-dimensions :dimensions :as grounded-values} :- ::ads/grounded-values](let [show (or show max-cards)dashboard (generate-base-dashboard base-context dashboard-template grounded-values)]@@ -756,7 +748,7 @@- Additional options such as how many cards to show, a cell query (a drill through), etc."[entity opts](automagic-analysis-method- (lib/normalize ::ads/root.entity entity)+ (m/assoc-some (lib/normalize ::ads/root.entity entity) :xrays/database-id (:xrays/database-id entity))(lib/normalize ::automagic-analysis.opts opts)))(defmethod automagic-analysis-method :model/Table@@ -764,7 +756,7 @@(automagic-dashboard (merge (->root table) opts)))(mu/defmethod automagic-analysis-method :model/Segment- [segment :- [:map [:definition ::segments.schema/definition]]+ [segment :- [:map [:definition [:maybe ::segments.schema/definition]]]opts](automagic-dashboard (merge (->root segment) opts)))@@ -777,8 +769,7 @@(ms/InstanceOf :xrays/Metric)::ads/metric]]][root :- ::ads/root- question :- [:map- [:dataset_query ::ads/query]]]+ question :- ::ads/card-or-question](map (mu/fn [aggregation-clause :- ::lib.schema.aggregation/aggregation](if (lib/clause-of-type? aggregation-clause :metric);; any [:metric ...] MBQL clauses these days are V2 Metrics and Automagic Dashboards do not handle them.@@ -792,8 +783,7 @@(mu/defn- collect-breakout-fields :- [:maybe [:sequential (ms/InstanceOf :model/Field)]][root :- ::ads/root- question :- [:map- [:dataset_query ::ads/query]]]+ question :- ::ads/card-or-question](for [breakout (lib/breakouts (:dataset_query question))field-clause (take 1 (magic.util/collect-field-references breakout)):let [field (magic.util/->field root field-clause)]@@ -803,9 +793,8 @@(mu/defn- decompose-question[root :- ::ads/root- question :- [:map- [:dataset_query ::ads/query]]- opts]+ question :- ::ads/card-or-question+ opts :- ::automagic-analysis.opts](letfn [(analyze [x](try(automagic-analysis@@ -825,10 +814,10 @@(mu/defn- preserve-entity-element"Ensure that elements of an original dataset query are preserved in dashcard queries."- [dashboard- entity- getter-fn- setter-fn]+ [dashboard :- ::ads/dashboard+ entity :- ::ads/card-or-question+ getter-fn :- ifn?+ setter-fn :- ifn?];; disable ref validation because X-Rays does stuff in a wacko manner, it adds a bunch of filters and whatever that;; use columns from joins before adding the joins themselves (same with expressions), which is technically invalid;; at the time it happens but ends up resulting in a valid query at the end of the day. Maybe one day we can reworkdiff --git a/src/metabase/xrays/automagic_dashboards/dashboard_templates.clj b/src/metabase/xrays/automagic_dashboards/dashboard_templates.cljindex eccc25a78ec..6e2696a9339 100644--- a/src/metabase/xrays/automagic_dashboards/dashboard_templates.clj+++ b/src/metabase/xrays/automagic_dashboards/dashboard_templates.clj@@ -15,6 +15,7 @@[metabase.util.i18n :as i18n][metabase.util.malli :as mu][metabase.util.malli.registry :as mr]+ [metabase.util.malli.schema :as ms][metabase.util.performance :as perf][metabase.util.yaml :as yaml][metabase.xrays.automagic-dashboards.populate :as populate]@@ -39,7 +40,12 @@(def ^:private Score[:int {:min 0, :max max-score}])-(def ^:private MBQL [:maybe [:sequential :any]])+(mr/def ::template-clause+ "A metric or filter clause in a dashboard template: a vector of strings, keywords, numbers, booleans, nils, and nested clauses, whose `[dimension X]` placeholders grounding later resolves."+ [:sequential [:or :string :keyword number? :boolean :nil [:ref ::template-clause]]])++(def ^:private MBQL+ [:maybe ::template-clause])(def ^:private Identifier[:string@@ -71,7 +77,7 @@(comp (with-defaults {:score max-score})(shorthand-definition :metric))}Identifier- [:map+ [:map {:closed true}[:metric MBQL][:score Score][:name {:optional true} LocalizedString]]])@@ -82,7 +88,7 @@(comp (with-defaults {:score max-score})(shorthand-definition :filter))}Identifier- [:map+ [:map {:closed true}[:filter MBQL][:score Score]]])@@ -144,7 +150,7 @@(comp (with-defaults {:score max-score})(shorthand-definition :field_type))}Identifier- [:map+ [:map {:closed true}[:field_type AppliesTo][:score Score][:links_to {:optional true} TableType]@@ -172,7 +178,7 @@(if (string? x)x(u/qualified-name x)))}]- [:* :map]])+ [:* ms/VisualizationSettings]])(def ^:private Width[:int {:min 1, :max populate/grid-width}])@@ -186,7 +192,7 @@{x {}}x))}Identifier- [:map [:aggregation {:optional true} :string]]])+ [:map {:closed true} [:aggregation {:optional true} :string]]])(def ^:private Card[:map-of@@ -196,7 +202,8 @@:height populate/default-card-height})}Identifier[:map- {:decode/dashboard-template (fn [x]+ {:closed true+ :decode/dashboard-template (fn [x](if (sequential? x)(into {} x)x))}@@ -225,7 +232,7 @@x(apply merge x)))}Identifier- [:map+ [:map {:closed true}[:title LocalizedString][:score :int][:comparison_title {:optional true} LocalizedString]@@ -307,7 +314,7 @@(def DashboardTemplate"Specification defining an automagic dashboard."[:and- [:map+ [:map {:closed true}[:title LocalizedString][:dashboard-template-name :string][:specificity :int]@@ -319,7 +326,7 @@[:metrics {:optional true} [:maybe [:sequential Metric]]][:filters {:optional true} [:maybe [:sequential Filter]]][:groups {:optional true} Groups]- [:indepth {:optional true} [:maybe [:sequential :any]]]+ [:indepth {:optional true} [:maybe [:sequential :string]]][:dashboard_filters {:optional true} [:maybe [:sequential {:decode/dashboard-template u/one-or-many} :string]]]][:fn {:error/message "Valid metrics references"} valid-metrics-references?][:fn {:error/message "Valid filters references"} valid-filters-references?]diff --git a/src/metabase/xrays/automagic_dashboards/filters.clj b/src/metabase/xrays/automagic_dashboards/filters.cljindex 75acdeab035..45bdd6eb94c 100644--- a/src/metabase/xrays/automagic_dashboards/filters.clj+++ b/src/metabase/xrays/automagic_dashboards/filters.clj@@ -120,9 +120,9 @@(mu/defn add-filters"Add up to `max-filters` filters to dashboard `dashboard`. The `dimensions` argument is a list of fields for which tocreate filters."- [dashboard :- ::ads/dashboard- dimensions- max-filters]+ [dashboard :- ::ads/dashboard+ dimensions :- [:sequential ::ads/field]+ max-filters :- :int](let [fks (when-let [table-ids (not-empty (set (keep (comp :table_id :card)(:dashcards dashboard))))](field/with-targets (xrays.db/fk-fields-for-tables table-ids)))]diff --git a/src/metabase/xrays/automagic_dashboards/interesting.clj b/src/metabase/xrays/automagic_dashboards/interesting.cljindex 7ed52e14b63..f44fb03a19a 100644--- a/src/metabase/xrays/automagic_dashboards/interesting.clj+++ b/src/metabase/xrays/automagic_dashboards/interesting.clj@@ -45,6 +45,7 @@[medley.core :as m][metabase.lib-be.core :as lib-be][metabase.lib.core :as lib]+ [metabase.lib.schema.common :as lib.schema.common][metabase.lib.schema.metadata :as lib.schema.metadata][metabase.lib.schema.ref :as lib.schema.ref][metabase.models.interface :as mi]@@ -157,12 +158,13 @@(mu/defn transform-metric-aggregate :- ::ads/external-op"Map a metric aggregate definition from nominal types to semantic types."- [[ag-type & args] dimension-name->field]+ [[ag-type & args] :- ::lib.schema.common/possibly-unnormalized-clause+ dimension-name->field :- [:map-of :string ::ads/item]]{:lib/type :lib/external-op:operator (keyword ag-type):args (mapv (fn [arg](if (and (vector? arg)- (= (first arg) "dimension"))+ (contains? #{:dimension "dimension"} (first arg)))(when-let [field (dimension-name->field (second arg))](field->metadata field))arg))@@ -284,7 +286,7 @@"For every field in a given context determine all potential dimensions each field may map to.This will return a map of field id (or name) to collection of potential matching dimensions."[context :- ::ads/context- dimension-specs]+ dimension-specs :- [:maybe [:sequential ::ads/dimension-template]]];; TODO - Fix this so that the intermediate representations aren't so crazy.;; all-bindings a map of binding dim identifier to binding def which contains;; field matches which are all the same field except they are merged with the binding.@@ -355,7 +357,8 @@(see `most-specific-definition` for details).The context is passed in, but it only needs tables and fields in `candidate-bindings`. It is not extensively used."- [context dimension-specs :- [:maybe [:sequential ::ads/dimension-template]]]+ [context :- ::ads/context+ dimension-specs :- [:maybe [:sequential ::ads/dimension-template]]](->> (candidate-bindings context dimension-specs)(map (comp most-specific-matched-dimension val))(apply merge-with (fn [a b]@@ -399,7 +402,8 @@(mu/defn grounded-filters :- [:sequential ::ads/grounded-filter]"Take filter templates (as from a dashboard template's :filters) and ground dimensions and produce a map of thefilter name to grounded versions of the filter."- [filter-templates ground-dimensions]+ [filter-templates :- [:maybe [:sequential ::ads/filter-template]]+ ground-dimensions :- ::ads/dim-name->matching-fields](->> filter-templates(keep (fn [fltr](let [[fname {:keys [filter] :as v}] (first fltr)@@ -416,7 +420,7 @@:operator (keyword op):args (mapv (fn [arg](if (and (vector? arg)- (= (first arg) "dimension"))+ (contains? #{:dimension "dimension"} (first arg)))(when-let [field (opt (second arg))](field->metadata field))arg))@@ -431,7 +435,7 @@[context :- ::ads/context{:keys [dimension-specsmetric-specs- filter-specs]} :- [:map+ filter-specs]} :- [:map {:closed true}[:dimension-specs [:maybe [:sequential ::ads/dimension-template]]][:metric-specs [:maybe [:sequential ::ads/metric-template]]][:filter-specs [:maybe [:sequential ::ads/filter-template]]]]]diff --git a/src/metabase/xrays/automagic_dashboards/names.clj b/src/metabase/xrays/automagic_dashboards/names.cljindex 98b01ed3048..2e5fa9f31cc 100644--- a/src/metabase/xrays/automagic_dashboards/names.clj+++ b/src/metabase/xrays/automagic_dashboards/names.clj@@ -52,7 +52,7 @@(mu/defn- join-enumeration :- ::ads/string-or-18n-string"Join a sequence as [1 2 3 4] to \"1, 2, 3 and 4\""- [xs :- [:sequential :any]]+ [xs :- [:sequential ::ads/string-or-18n-string]](if (next xs)(tru "{0} and {1}" (str/join ", " (butlast xs)) (last xs))(str (first xs))))@@ -83,8 +83,7 @@(mu/defn question-description"Generate a description for the question."[root :- ::ads/root- question :- [:map- [:dataset_query ::ads/query]]]+ question :- ::ads/card-or-question](let [aggregations (->> (lib/aggregations (:dataset_query question))(metric->description root))field-ids (intodiff --git a/src/metabase/xrays/automagic_dashboards/populate.clj b/src/metabase/xrays/automagic_dashboards/populate.cljindex b341cff9e44..b6cecf24c9c 100644--- a/src/metabase/xrays/automagic_dashboards/populate.clj+++ b/src/metabase/xrays/automagic_dashboards/populate.clj@@ -132,7 +132,7 @@"Add a card to dashboard `dashboard` at position [`x`, `y`]."[dashboard :- ::ads/dashboard{query :dataset_query, :keys [title description width height id] :as dashcard} :- ::ads/card-template- [x y]]+ [x y] :- [:tuple nat-int? nat-int?]](let [query-fields (when query;; disable ref validation because X-Rays does stuff in a wacko manner, it adds a bunch of;; filters and whatever that use columns from joins before adding the joins themselves (same@@ -190,9 +190,7 @@occupied."[grid :- ::grid[x y] :- [:tuple nat-int? nat-int?]- {:keys [width height], :as _dashcard} :- [:map- [:width nat-int?]- [:height nat-int?]]]+ {:keys [width height], :as _dashcard} :- ::ads/card-template](reduce (fn [grid xy](assoc-in grid xy true))grid@@ -206,9 +204,7 @@it suffices to check just the first (top) row."[grid :- ::grid[x y] :- [:tuple nat-int? nat-int?]- {:keys [width height]} :- [:map- [:width nat-int?]- [:height nat-int?]]]+ {:keys [width height]} :- ::ads/card-template](and (<= (+ x height) (count grid))(<= (+ y width) (-> grid first count))(every? false? (subvec (grid x) y (+ y width)))))@@ -219,9 +215,9 @@we should be fine): starting at top left move along the grid from left toright, row by row and try to place the card at each position until we find anunoccupied area. Mark the area as occupied."- [grid :- ::grid- start-row- dashcard]+ [grid :- ::grid+ start-row :- nat-int?+ dashcard :- ::ads/card-template](reduce (fn [grid xy](if (accommodates? grid xy dashcard)(reduced xy)@@ -251,7 +247,11 @@(mu/defn- add-group :- [:tuple ::ads/dashboard ::grid][dashboard :- ::ads/dashboardgrid :- ::grid- group+ group :- [:maybe [:map {:closed true}+ [:title ::ads/string-or-18n-string]+ [:score {:optional true} :int]+ [:comparison_title {:optional true} [:maybe ::ads/string-or-18n-string]]+ [:description {:optional true} [:maybe ::ads/string-or-18n-string]]]]cards :- [:sequential ::ads/card-template]](let [start-row (bottom-row grid)start-row (cond-> start-row@@ -314,7 +314,7 @@(mu/defn create-dashboard :- ::ads/dashboard"Create dashboard and populate it with cards."- ([dashboard] (create-dashboard dashboard :all))+ ([dashboard :- ::ads/dashboard-template] (create-dashboard dashboard :all))([{:keys [title transient_title description groups filters cards]} :- ::ads/dashboard-templaten :- [:or pos-int? :keyword]](let [n (conddiff --git a/src/metabase/xrays/automagic_dashboards/schema.clj b/src/metabase/xrays/automagic_dashboards/schema.cljindex 24208027b9f..a42b3e806a1 100644--- a/src/metabase/xrays/automagic_dashboards/schema.clj+++ b/src/metabase/xrays/automagic_dashboards/schema.clj@@ -1,14 +1,16 @@(ns metabase.xrays.automagic-dashboards.schema(:require- [malli.core :as mc][metabase.lib.core :as lib][metabase.lib.schema :as lib.schema][metabase.lib.schema.aggregation :as lib.schema.aggregation][metabase.lib.schema.common :as lib.schema.common][metabase.lib.schema.expression :as lib.schema.expression][metabase.lib.schema.id :as lib.schema.id]+ [metabase.lib.schema.literal :as lib.schema.literal][metabase.lib.schema.mbql-clause :as lib.schema.mbql-clause][metabase.lib.schema.metadata :as lib.schema.metadata]+ [metabase.queries.schema]+ [metabase.segments.schema][metabase.util.i18n :as i18n][metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms]@@ -17,12 +19,24 @@(mr/def ::string-or-18n-string[:or :string [:fn {:error/message "localized string"} i18n/localized-string?]])+(mr/def ::related-keys+ "What [[metabase.xrays.related]] and the candidate-table ranking pin on an entity they return."+ [:map {:closed true}+ [:similarity {:optional true} [:maybe number?]]+ [:num-fields {:optional true} [:maybe :int]]+ [:list-like? {:optional true} [:maybe [:or :boolean :int]]]+ [:link {:optional true} [:maybe ::lib.schema.id/field]]])+(mr/def ::root.entity[:multi{:dispatch t2/model}- [:xrays/Metric [:ref ::metric]]- [::mc/default [:map- [:name {:optional true} :string]]]])+ [:xrays/Metric [:ref ::metric]]+ [:model/Table [:merge :metabase.warehouse-schema.schema/table [:ref ::related-keys]]]+ [:model/Field [:ref ::field]]+ [:model/Segment [:merge :metabase.segments.schema/segment [:ref ::related-keys]]]+ [:model/Card [:merge :metabase.queries.schema/card [:ref ::related-keys]+ [:map [:entity_type {:optional true} :keyword]]]]+ [:model/Query [:ref ::adhoc-question]]])(mr/def ::filter-clause[:and@@ -37,43 +51,80 @@(mr/def ::root.cell-query::filter-clause)+(mr/def ::root.cell-query-input+ "A `::root.cell-query` before [[metabase.xrays.automagic-dashboards.comparison/comparison-dashboard]] and friends+ normalize it with `(lib/normalize ::root.cell-query ...)`: a raw, possibly-legacy filter clause as callers (API+ params decoded from JSON, or a hand-written test clause) actually pass it in."+ ::lib.schema.common/possibly-unnormalized-clause)+(mr/def ::root- [:map- [:database ::lib.schema.id/database]- [:entity {:optional true} [:ref ::root.entity]]- [:query-filter {:optional true} [:maybe [:sequential ::filter-clause]]]- [:cell-query {:optional true} [:maybe [:ref ::root.cell-query]]]])+ [:map {:closed true}+ [:database ::lib.schema.id/database]+ [:entity {:optional true} [:ref ::root.entity]]+ [:query-filter {:optional true} [:maybe [:sequential ::filter-clause]]]+ [:cell-query {:optional true} [:maybe [:ref ::root.cell-query]]]+ [:full-name {:optional true} [:maybe ::string-or-18n-string]]+ [:short-name {:optional true} [:maybe ::string-or-18n-string]]+ [:comparison-name {:optional true} [:maybe ::string-or-18n-string]]+ [:source {:optional true} [:maybe ::source]]+ [:url {:optional true} :string]+ [:dashboard-templates-prefix {:optional true} [:sequential :string]]+ [:comparison? {:optional true} [:maybe :boolean]]+ [:rules-prefix {:optional true} [:maybe [:sequential :string]]]+ [:dashboard-template {:optional true} [:maybe [:sequential :string]]]+ [:show {:optional true} [:maybe [:or pos-int? [:= :all]]]]])(mr/def ::source[:or(ms/InstanceOf #{:model/Table :model/Card})- [:map+ [:map {:closed true}[:entity_type [:andqualified-keyword?[:fn{:error/message ":entity/ keyword"}- #(= (namespace %) "entity")]]]]])+ #(= (namespace %) "entity")]]]+ [:fields {:optional true} [:sequential ::field]]]])(mr/def ::context"The big ball of mud data object from which we generate x-rays"- [:map+ [:map {:closed true}[:source {:optional true} ::source][:root {:optional true} [:ref ::root]]- [:tables {:optional true} any?]+ [:tables {:optional true} [:sequential ::source]][:query-filter {:optional true} [:maybe [:sequential ::filter-clause]]]])(mr/def ::query"Schema for the type of MBQL queries handled by X-Rays."[:ref ::lib.schema/query])+(mr/def ::adhoc-question+ "The ad-hoc \"question\" wrapper automagic-dashboards builds for a raw query that is not backed by a saved Card+ (see [[metabase.xrays.api.automagic-dashboards/adhoc-query-instance]])."+ [:map {:closed true}+ [:dataset_query ::query]+ [:database-id {:optional true} ::lib.schema.id/database]+ [:table-id {:optional true} [:maybe ::lib.schema.id/table]]+ [:entity_type {:optional true} :keyword]])++(mr/def ::card-or-question+ "Either a Card row, or the [[::adhoc-question]] wrapper for a raw query. Several helpers in+ [[metabase.xrays.automagic-dashboards.core]] duck-type over both. Either may carry the `:entity_type`+ [[metabase.xrays.automagic-dashboards.core/source]] assoc's on."+ [:or+ [:merge :metabase.queries.schema/card [:map [:entity_type {:optional true} :keyword]]]+ ::adhoc-question])+(mr/def ::external-op[:merge::lib.schema.common/external-op[:map- [:args [:sequential [:multi- {:dispatch coll?, :error/message "Should be a literal or column metadata"}- [false :any]- [true ::lib.schema.metadata/column]]]]]])+ [:args [:sequential+ [:multi+ {:dispatch (fn [x] (cond (map? x) :column, (sequential? x) :aggregation, :else :literal))+ :error/message "Should be a literal, column metadata, or a nested aggregation clause"}+ [:literal [:or ::lib.schema.literal/param-value :keyword]]+ [:column ::lib.schema.metadata/column]+ [:aggregation [:ref ::aggregation]]]]]]])(mr/def ::aggregation[:or ::lib.schema.aggregation/aggregation ::external-op])@@ -99,11 +150,12 @@(mr/def ::dimension-value"A specification for the basic keys in the value of a dimension template."- [:map+ [:map {:closed true}[:field_type ::field-type][:score {:optional true} nat-int?][:max_cardinality {:optional true} nat-int?]- [:named {:optional true} [:string {:min 1}]]])+ [:named {:optional true} [:string {:min 1}]]+ [:links_to {:optional true} :keyword]])(mr/def ::dimension-template"A specification for the basic keys in a dimension template."@@ -116,7 +168,7 @@"A specification for the basic keys in the value of a metric template."[:map{:closed true}- [:metric [:vector some?]]+ [:metric ::lib.schema.common/possibly-unnormalized-clause][:score {:optional true} nat-int?][:name {:optional true} ::string-or-18n-string]])@@ -129,8 +181,8 @@(mr/def ::filter-value"A specification for the basic keys in the value of a filter template."- [:map- [:filter [:vector some?]]+ [:map {:closed true}+ [:filter ::lib.schema.common/possibly-unnormalized-clause][:score nat-int?]])(mr/def ::filter-template@@ -141,10 +193,9 @@::filter-value])(mr/def ::item- "A \"thing\" that we bind to, consisting, generally, of at least a name and id"- [:map- [:id {:optional true} nat-int?]- [:name {:optional true} string?]])+ "A \"thing\" that we bind to: the same shape as [[::field]], a source row merged with the dimension-template and+ grounding keys that matched it."+ ::field)(mr/def ::dim-name->dim-def"A map of dimension name to dimension definition."@@ -155,15 +206,20 @@and a sequence of matching items satisfying this dimension"[:map-of:string- [:map- [:matches [:sequential ::item]]]])+ [:map {:closed true}+ [:matches [:sequential ::item]]+ [:field_type {:optional true} ::field-type]+ [:score {:optional true} nat-int?]+ [:max_cardinality {:optional true} nat-int?]+ [:named {:optional true} [:string {:min 1}]]+ [:links_to {:optional true} :keyword]+ [:name {:optional true} :string]+ [:card-score {:optional true} number?]]])(mr/def ::dim-name->dim-defs+matches"The \"full\" grounded dimensions which matches dimension namesto the dimension definition combined with matching fields."- [:merge- ::dim-name->dim-def- ::dim-name->matching-fields])+ ::dim-name->matching-fields)(mr/def ::normalized-metric-template"A \"normalized\" metric template is a map containing the metric name as a key@@ -173,7 +229,7 @@[:name {:optional true} ::string-or-18n-string][:metric-name :string][:score nat-int?]- [:metric vector?]])+ [:metric ::lib.schema.common/possibly-unnormalized-clause]])(mr/def ::grounded-metric.definition[:map@@ -208,16 +264,17 @@[:card-name {:optional true} :string][:height {:optional true} number?][:width {:optional true} number?]- [:title {:optional true} :string]- [:visualization {:optional true} [:tuple :string :map]]+ [:title {:optional true} [:maybe ::string-or-18n-string]]+ [:visualization {:optional true} [:tuple :string ms/VisualizationSettings]][:metrics {:optional true} [:sequential :string]][:filters {:optional true} [:sequential :string]]- [:description {:optional true} :string]- [:dimensions {:optional true} [:sequential [:map-of :string :map]]]+ [:description {:optional true} [:maybe ::string-or-18n-string]]+ [:dimensions {:optional true} [:sequential [:map-of :string [:map {:closed true}+ [:aggregation {:optional true} :string]]]]];; HUH??[:order_by {:optional true} [:sequential [:map-of :string [:enum "ascending" "descending"]]]][:limit {:optional true} pos-int?]- [:x_label {:optional true} :string]+ [:x_label {:optional true} [:maybe ::string-or-18n-string]][:metric-definition[:merge::grounded-metric.definition@@ -231,23 +288,62 @@[:filter ::external-op][:filter-name :string]])+(mr/def ::field.xray-keys+ "What X-Rays pins on a Field while grounding it to a dimension: the matched dimension definition, the Database, the+ FK it was reached through, and the scores it ranks it by."+ [:map {:closed true}+ [:db {:optional true} [:maybe [:ref :metabase.warehouses.schema/database]]]+ [:link {:optional true} [:maybe ::lib.schema.id/field]]+ [:aggregation {:optional true} [:maybe :string]]+ [:field_type {:optional true} [:maybe ::field-type]]+ [:links_to {:optional true} [:maybe :keyword]]+ [:named {:optional true} [:maybe :string]]+ [:max_cardinality {:optional true} [:maybe nat-int?]]+ [:score {:optional true} [:maybe number?]]+ [:card-score {:optional true} [:maybe number?]]+ [:interestingness {:optional true} [:maybe number?]]+ [:similarity {:optional true} [:maybe number?]]+ [:xrays/database-id {:optional true} [:maybe ::lib.schema.id/database]]])+(mr/def ::field- [:and- [:map- ;; as mentioned elsewhere X-Rays does some kind of insane nonsense and creates fields with types like- ;; `:type/GenericNumber` when instantiating templates- [:base_type {:optional true} ::lib.schema.common/base-type]]- [:fn- {:error/message "Should be a field with snake_case keys"}- (complement :base-type)]])+ "A Field X-Rays binds to a dimension: a Field row, or a Card's result metadata column wrapped as a Field, plus the+ keys grounding pins on it."+ [:multi {:dispatch (fn [field]+ (if (contains? field :created_at) :row :result-column))}+ [:row [:merge :metabase.warehouse-schema.schema/field [:ref ::field.xray-keys]]]+ [:result-column [:merge :metabase.legacy-mbql.schema/legacy-column-metadata [:ref ::field.xray-keys]]]])(mr/def ::card- [:map- [:id {:optional true} [:or symbol? ::lib.schema.id/card]]- [:dataset_query {:optional true} ::query]])+ "A \"card\" as it flows through the dashboard-building pipeline: the keys [[metabase.xrays.automagic-dashboards+ .populate/add-normal-dashcard]] builds it with, plus the render-stage additions+ [[metabase.xrays.automagic-dashboards.comparison/dashboard->cards]] assocs onto it."+ [:map {:closed true}+ [:id {:optional true} [:or symbol? ::lib.schema.id/card]]+ [:dataset_query {:optional true} ::query]+ [:creator_id {:optional true} [:maybe ::lib.schema.id/user]]+ [:description {:optional true} [:maybe :string]]+ [:name {:optional true} [:maybe :string]]+ [:collection_id {:optional true} [:maybe ::lib.schema.id/collection]]+ [:display {:optional true} [:maybe [:or :keyword :string]]]+ [:visualization_settings {:optional true} ms/VisualizationSettings]+ [:query_type {:optional true} [:maybe [:or :keyword :string]]]+ [:database_id {:optional true} [:maybe ::lib.schema.id/database]]+ [:table_id {:optional true} [:maybe ::lib.schema.id/table]]+ [:text {:optional true} [:maybe :string]]+ [:series {:optional true} [:maybe [:sequential [:ref ::card]]]]+ [:height {:optional true} number?]+ [:position {:optional true} number?]])++(mr/def ::parameter-mapping+ "One entry of a dashcard's `:parameter_mappings`, as+ [[metabase.xrays.automagic-dashboards.filters/add-filter]] builds it."+ [:map {:closed true}+ [:parameter_id :string]+ [:target ::lib.schema.common/possibly-unnormalized-clause]+ [:card_id {:optional true} [:or symbol? ::lib.schema.id/card]]])(mr/def ::dashcard- [:map+ [:map {:closed true}[:id {:optional true} [:or symbol? ::lib.schema.id/dashcard]][:card {:optional true} ::card][:card_id {:optional true} [:or symbol? ::lib.schema.id/card]]@@ -255,23 +351,106 @@[:row {:optional true} nat-int?][:size_x {:optional true} pos-int?][:size_y {:optional true} pos-int?]- [:visualization_settings {:optional true} map?]+ [:visualization_settings {:optional true} ms/VisualizationSettings][:title {:optional true} string?]- [:card-score {:optional true} number?]])+ [:card-score {:optional true} number?]+ [:dashboard_tab_id {:optional true} [:maybe :int]]+ [:creator_id {:optional true} [:maybe ::lib.schema.id/user]]+ [:series {:optional true} [:maybe [:sequential ::card]]]+ [:parameter_mappings {:optional true} [:maybe [:sequential ::parameter-mapping]]]])++(mr/def ::dashboard-parameter+ "A filter widget [[metabase.xrays.automagic-dashboards.filters/add-filters]] adds to a dashboard."+ [:map {:closed true}+ [:id :string]+ [:name [:maybe :string]]+ [:slug :string]+ [:type :string]+ [:sectionId :string]])++(mr/def ::related-entry+ "One entry in a [[::related]] bucket: a link to another x-ray."+ [:map {:closed true}+ [:url :string]+ [:title ::string-or-18n-string]+ [:description [:maybe ::string-or-18n-string]]])++(mr/def ::related+ "The `:related` links of a populated dashboard: up to 4 buckets (which ones depend on the entity's model), each a+ list of links round-robined from candidates like segments, tables, or drilldown fields."+ [:map {:closed true}+ [:zoom-in {:optional true} [:sequential ::related-entry]]+ [:zoom-out {:optional true} [:sequential ::related-entry]]+ [:related {:optional true} [:sequential ::related-entry]]+ [:compare {:optional true} [:sequential ::related-entry]]])(mr/def ::dashboard- [:map- [:dashcards {:optional true} [:sequential ::dashcard]]- [:filters {:optional true} [:sequential :any]]])+ [:map {:closed true}+ [:name {:optional true} ::string-or-18n-string]+ [:transient_name {:optional true} [:maybe ::string-or-18n-string]]+ [:description {:optional true} [:maybe ::string-or-18n-string]]+ [:creator_id {:optional true} [:maybe ::lib.schema.id/user]]+ [:parameters {:optional true} [:maybe [:sequential ::dashboard-parameter]]]+ [:dashcards {:optional true} [:maybe [:sequential ::dashcard]]]+ [:filters {:optional true} [:maybe [:sequential ::item]]]+ [:related {:optional true} ::related]+ [:more {:optional true} [:maybe :string]]+ [:transient_filters {:optional true} [:maybe [:sequential ::filter-clause]]]+ [:param_fields {:optional true} [:maybe [:map-of :string [:sequential ::item]]]]+ [:auto_apply_filters {:optional true} :boolean]+ [:width {:optional true} [:enum "fixed" "full"]]])++(mr/def ::transform-card.layout+ "The section layout keys [[metabase.xrays.transforms.dashboard]] assocs onto each card it lays out."+ [:map {:closed true}+ [:group :string]+ [:width number?]+ [:height number?]+ [:card-score number?]+ [:position number?]+ [:title {:optional true} [:maybe :string]]+ [:visualization {:optional true} [:tuple [:or :keyword :string]]]+ [:text {:optional true} [:maybe :string]]])++(mr/def ::transform-card+ "A card [[metabase.xrays.transforms.dashboard]] lays out: a saved Card, or an unsaved source-table or text card, with its section layout keys."+ [:or+ [:merge :metabase.queries.schema/card ::transform-card.layout]+ [:merge :metabase.queries.schema/card.update ::transform-card.layout]])(mr/def ::card-template- :map)+ "A grounded, combined metric augmented with the extra keys the dashboard-populating code+ ([[metabase.xrays.automagic-dashboards.populate]]) reads off a card before rendering it, or a card a transform+ dashboard lays out. A plain text/group-heading card carries none of the metric keys, so they're all optional here."+ [:or+ [:merge+ ::combined-metric+ [:map {:closed true}+ [:metric-name {:optional true} :string]+ [:metric-title {:optional true} :string]+ [:metric-score {:optional true} nat-int?]+ [:metric-definition {:optional true} ::grounded-metric.definition]+ [:dataset_query {:optional true} ::query]+ [:y_label {:optional true} :string]+ [:series_labels {:optional true} [:sequential :string]]+ [:text {:optional true} :string]+ [:visualization-settings {:optional true} ms/VisualizationSettings]]]+ ::transform-card])(mr/def ::dashboard-template"This is somewhat different [[metabase.xrays.automagic-dashboards.schema/DashboardTemplate]], I haven't exactly workedout what the schema is supposed to be yet."- [:map- [:cards {:optional true} [:maybe [:sequential ::card-template]]]])+ [:map {:closed true}+ [:cards {:optional true} [:maybe [:sequential ::card-template]]]+ [:title {:optional true} [:maybe ::string-or-18n-string]]+ [:transient_title {:optional true} [:maybe ::string-or-18n-string]]+ [:description {:optional true} [:maybe ::string-or-18n-string]]+ [:filters {:optional true} [:sequential ::item]]+ [:groups {:optional true} [:maybe [:map-of :string [:map {:closed true}+ [:title {:optional true} ::string-or-18n-string]+ [:score {:optional true} :int]+ [:comparison_title {:optional true} [:maybe ::string-or-18n-string]]+ [:description {:optional true} [:maybe ::string-or-18n-string]]]]]]])(mr/def ::grounded-values[:mapdiff --git a/src/metabase/xrays/automagic_dashboards/util.clj b/src/metabase/xrays/automagic_dashboards/util.cljindex 377cae70650..ed6f877f84d 100644--- a/src/metabase/xrays/automagic_dashboards/util.clj+++ b/src/metabase/xrays/automagic_dashboards/util.clj@@ -5,6 +5,7 @@[medley.core :as m][metabase.analyze.core :as analyze][metabase.lib.core :as lib]+ [metabase.lib.schema.common :as lib.schema.common][metabase.lib.schema.id :as lib.schema.id][metabase.models.interface :as mi][metabase.types.core :as types]@@ -20,7 +21,7 @@(mu/defn field-isa?"`isa?` on a field, checking semantic_type and then base_type"- [{:keys [base_type semantic_type]} :- ::ads/field+ [{:keys [base_type semantic_type]} :- [:or ::ads/field ::ads/root.entity];; for some insane reason this is called with totally imaginary types like `:type/GenericNumber`t :- [:andqualified-keyword?@@ -42,7 +43,8 @@(mu/defn filter-tables :- [:sequential ::ads/source]"filter `tables` by `tablespec`, which is just an entity type (eg. :entity/GenericTable)"- [tablespec tables :- [:maybe [:sequential ::ads/source]]]+ [tablespec :- :keyword+ tables :- [:maybe [:sequential ::ads/source]]](filter #(isa? types/entity-hierarchy (:entity_type %) tablespec) tables))(defn ancestor-count@@ -74,7 +76,9 @@(mu/defn collect-field-references :- [:maybe [:sequential :mbql.clause/field]]"Collect all `:field` references from a given form."- [form]+ [form :- [:or+ ::lib.schema.common/possibly-unnormalized-clause+ [:sequential ::lib.schema.common/possibly-unnormalized-clause]]](match/match-many form [:field & _] &match))(mu/defn ->field :- [:maybe [:and@@ -104,8 +108,8 @@(update :base_type keyword)(update :semantic_type keyword)(->> (mi/instance :model/Field))- (assoc :xrays/database-id (:database root))- (analyze/run-classifiers {}))))+ (analyze/run-classifiers {})+ (assoc :xrays/database-id (:database root)))));; otherwise this isn't returning something, and that's probably an error. Log it.(log/warnf "Cannot resolve Field %s in automagic analysis context" field-id-or-name-or-clause))))diff --git a/src/metabase/xrays/db.clj b/src/metabase/xrays/db.cljindex 6a4860459b0..dd7295d14ab 100644--- a/src/metabase/xrays/db.clj+++ b/src/metabase/xrays/db.clj@@ -257,7 +257,7 @@(mu/defn other-card-ids-on-dashboards"The Card ids other than `card-id` of the DashboardCards of the Dashboards with `dashboard-ids`."- [dashboard-ids :- [:sequential ::lib.schema.id/dashboard]+ [dashboard-ids :- [:set ::lib.schema.id/dashboard]card-id :- ::lib.schema.id/card](t2/select-fn-set :card_id :model/DashboardCard :dashboard_id [:in dashboard-ids] :card_id [:not= card-id]))diff --git a/src/metabase/xrays/domain_entities/core.clj b/src/metabase/xrays/domain_entities/core.cljindex 7a84625f4e6..f47c32798d4 100644--- a/src/metabase/xrays/domain_entities/core.clj+++ b/src/metabase/xrays/domain_entities/core.clj@@ -1,24 +1,32 @@(ns metabase.xrays.domain-entities.core(:require[clojure.string :as str]+ [malli.core :as mc][medley.core :as m];; legacy usage, do not use legacy MBQL stuff in new code.^{:clj-kondo/ignore [:deprecated-namespace :discouraged-namespace]} [metabase.legacy-mbql.util :as mbql.u][metabase.lib.schema.common :as lib.schema.common][metabase.util :as u][metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms][metabase.util.match :as match]+ [metabase.warehouse-schema.schema][metabase.xrays.domain-entities.hierarchy :as domain-entities.hierarchy][metabase.xrays.domain-entities.specs :as domain-entities.specs :refer [*domain-entity-specs* MBQL]][toucan2.core :as t2]))+(mr/def ::field-or-column+ "A Field row, or the column metadata of a Card's `:result_metadata` or a query's expected columns."+ [:multi {:dispatch t2/model}+ [:model/Field :metabase.warehouse-schema.schema/field]+ [::mc/default :metabase.lib.schema.metadata/lib-or-legacy-column]])+(mu/defn field-type :- [:or::lib.schema.common/base-type::lib.schema.common/semantic-or-relation-type]"Return the most specific type of a given field."- [field :- [:map- [:base_type ::lib.schema.common/base-type]]]+ [field :- ::field-or-column]((some-fn :semantic_type :base_type) field))(def SourceName@@ -40,10 +48,19 @@`DimensionBindings`."[:map-ofSourceName- [:map+ [:map {:closed true}[:dimensions DimensionBindings][:entity {:optional true} SourceEntity]]])+(def ^:private MetricOrSegmentEntry+ "The value half of a domain-entity spec's `:metrics`/`:segments` map entries."+ [:map {:closed true}+ [:name :string]+ [:aggregation {:optional true} MBQL]+ [:breakout {:optional true} [:sequential MBQL]]+ [:filter {:optional true} MBQL]+ [:description {:optional true} :string]])+(mu/defn- get-dimension-binding :- MBQL[bindings :- Bindingssource :- SourceName@@ -61,7 +78,7 @@"Instantiate all dimension reference in given (nested) structure"[bindings :- Bindingssource :- SourceName- obj]+ obj :- [:maybe [:or MBQL [:tuple :string MetricOrSegmentEntry]]]](match/replace obj[:dimension dimension] (->> dimension(get-dimension-binding bindings source)@@ -69,8 +86,7 @@(mu/defn mbql-reference :- MBQL"Return MBQL clause for a given field-like object."- [{:keys [id name base_type]} :- [:map- [:base_type ::lib.schema.common/base-type]]]+ [{:keys [id name base_type]} :- ::field-or-column](if id[:field id nil][:field name {:base-type base_type}]))diff --git a/src/metabase/xrays/domain_entities/queries/util.cljc b/src/metabase/xrays/domain_entities/queries/util.cljcindex bd8417628d6..41e44e19793 100644--- a/src/metabase/xrays/domain_entities/queries/util.cljc+++ b/src/metabase/xrays/domain_entities/queries/util.cljc@@ -2,11 +2,12 @@"Utility functions used by the Queries in metabase-lib."(:require#?@(:cljs ([metabase.xrays.domain-entities.converters :as converters]))+ [metabase.lib.schema.common :as lib.schema.common][metabase.util.malli :as mu]))(def Expression"Schema for an Expression that's part of a query filter."- :any)+ ::lib.schema.common/possibly-unnormalized-clause)(def ExpressionMap"Malli schema for a map of expressions by name."diff --git a/src/metabase/xrays/domain_entities/specs.clj b/src/metabase/xrays/domain_entities/specs.cljindex 4ae1610843d..cbd8c06cb71 100644--- a/src/metabase/xrays/domain_entities/specs.clj+++ b/src/metabase/xrays/domain_entities/specs.clj@@ -82,10 +82,10 @@(def ^:private Description :string)(mr/def ::attribute- [:map+ [:map {:closed true}[:field {:optional true} BrokenFieldNameTypeKeyword][:domain_entity {:optional true} DomainEntityReference]- [:has_many {:optional true} [:map+ [:has_many {:optional true} [:map {:closed true}[:domain_entity DomainEntityReference]]]])(mr/def ::attributes@@ -111,7 +111,7 @@[:map-of{:decode/domain-entity-spec add-name-from-key}Identifier- [:map+ [:map {:closed true}[:aggregation MBQL][:name Identifier][:breakout {:optional true} BreakoutDimensions]@@ -122,14 +122,14 @@[:map-of{:decode/domain-entity-spec add-name-from-key}Identifier- [:map+ [:map {:closed true}[:filter MBQL][:name Identifier][:description {:optional true} Description]]])(def DomainEntitySpec"Domain entity spec"- [:map+ [:map {:closed true}[:name DomainEntityReference][:type DomainEntityType][:required_attributes ::attributes]diff --git a/src/metabase/xrays/related.clj b/src/metabase/xrays/related.cljindex 8b626696603..d189fa48cbf 100644--- a/src/metabase/xrays/related.clj+++ b/src/metabase/xrays/related.clj@@ -62,7 +62,7 @@nil)))(mu/defmethod definition :model/Segment- [segment :- [:map [:definition ::segments.schema/definition]]]+ [segment :- [:map [:definition [:maybe ::segments.schema/definition]]]](-> segment :definition :stages first :filters not-empty))(defmethod definition :model/Field@@ -138,7 +138,7 @@(defn- cards-sharing-dashboard[card]- (if-let [dashboards (not-empty (xrays.db/dashboard-ids-for-card (:id card)))]+ (if-let [dashboards (some-> (:id card) xrays.db/dashboard-ids-for-card not-empty)](->> (xrays.db/other-card-ids-on-dashboards dashboards (:id card))(map xrays.db/card)filter-visiblediff --git a/src/metabase/xrays/transforms/core.clj b/src/metabase/xrays/transforms/core.cljindex 71217d99e3e..71544658ad9 100644--- a/src/metabase/xrays/transforms/core.clj+++ b/src/metabase/xrays/transforms/core.clj@@ -142,8 +142,10 @@[:sequential[:and(ms/InstanceOf :model/Table)- [:map- [:domain_entity [:maybe ::domain-entities.specs/instantiated-domain-entity]]]]])+ [:merge+ :metabase.warehouse-schema.schema/table+ [:map {:closed true}+ [:domain_entity [:maybe ::domain-entities.specs/instantiated-domain-entity]]]]]])(mu/defn- find-tables-with-domain-entity :- Tableset[tableset :- Tablesetdiff --git a/src/metabase/xrays/transforms/dashboard.clj b/src/metabase/xrays/transforms/dashboard.cljindex f37fe2820cf..3e2e3de48c2 100644--- a/src/metabase/xrays/transforms/dashboard.clj+++ b/src/metabase/xrays/transforms/dashboard.clj@@ -5,6 +5,7 @@[metabase.lib.schema.id :as lib.schema.id][metabase.util :as u][metabase.util.malli :as mu]+ [metabase.util.malli.schema :as ms][metabase.xrays.automagic-dashboards.populate :as populate][metabase.xrays.db :as xrays.db][metabase.xrays.transforms.materialize :as tf.materialize]@@ -15,9 +16,23 @@(def ^:private ^:const ^Long total-width 18)(def ^:private ^:const ^Long height 4)+(def ^:private SourceTableCard+ "The shape [[card-for-source-table]] builds for a source-table pseudo-card."+ [:map {:closed true}+ [:creator_id [:maybe ::lib.schema.id/user]]+ [:dataset_query [:map {:closed true}+ [:type [:= :query]]+ [:query [:map {:closed true} [:source-table ::lib.schema.id/table]]]+ [:database ::lib.schema.id/database]]]+ [:name [:maybe :string]]+ [:collection_id :nil]+ [:visualization_settings ms/VisualizationSettings]+ [:display :keyword]])+(mu/defn- cards->section"Build a section of cards and format them according to what the automagic dashboards code expects."- [group :- :string cards]+ [group :- :string+ cards :- [:maybe [:sequential [:or :metabase.queries.schema/card SourceTableCard]]]](mapcat (fn [{:keys [name description display] :as card}](cond-> [(assoc card:group group@@ -36,8 +51,7 @@cards))(mu/defn- card-for-source-table- [table :- [:map- [:db_id ::lib.schema.id/database]]]+ [table :- :metabase.warehouse-schema.schema/table]{:pre [(map? table)]}{:creator_id api/*current-user-id*:dataset_query {:type :querydiff --git a/src/metabase/xrays/transforms/materialize.clj b/src/metabase/xrays/transforms/materialize.cljindex aac8d2fabc8..59e6b396d71 100644--- a/src/metabase/xrays/transforms/materialize.clj+++ b/src/metabase/xrays/transforms/materialize.clj@@ -21,7 +21,7 @@(mu/defn get-collection :- [:maybe ::lib.schema.id/collection]"Get collection named `collection-name`. If no location is given root collection for automaticallygenerated transforms is assumed (see `get-or-create-root-container-collection!`)."- ([collection-name]+ ([collection-name :- :string](get-collection collection-name (root-container-location)))([collection-name :- :stringlocation :- :string]diff --git a/src/metabase/xrays/transforms/specs.clj b/src/metabase/xrays/transforms/specs.cljindex 1d955bed040..1759d52bf07 100644--- a/src/metabase/xrays/transforms/specs.clj+++ b/src/metabase/xrays/transforms/specs.clj@@ -72,7 +72,7 @@(def ^:private Joins[:sequential- [:map+ [:map {:closed true}[:source Source][:condition MBQL][:strategy {:optional true} JoinStrategy]]])@@ -81,7 +81,7 @@(def Step"Transform step"- [:map+ [:map {:closed true}[:source Source][:name Source][:transform TransformName]@@ -117,7 +117,7 @@(def TransformSpec"Transform spec"- [:map+ [:map {:closed true}[:name TransformName][:requires Requires][:provides Provides]diff --git a/test/metabase/actions/actions_test.clj b/test/metabase/actions/actions_test.cljindex 1a95c402d51..f0e7e30ecf5 100644--- a/test/metabase/actions/actions_test.clj+++ b/test/metabase/actions/actions_test.clj@@ -39,7 +39,7 @@(mu/defn- format-field-name :- :string"Format `field-name` appropriately for the current driver (e.g. uppercase it if we're testing against H2)."- [field-name]+ [field-name :- :keyword](name (mt/format-name (name field-name))))(defn- categories-row-count []@@ -86,7 +86,7 @@(is (= {:rows-updated 1}(actions/perform-action! :model.row/update(assoc (mt/mbql-query categories {:filter [:= $id 50]})- :update_row {(format-field-name :name) "updated_row"})))+ :update-row {(format-field-name :name) "updated_row"})))"Update should return the right shape")(is (= "updated_row"(-> (mt/rows (mt/run-mbql-query categories {:filter [:= $id 50]})) last last))@@ -120,14 +120,14 @@result)))}{:action :model.row/update:request-body (assoc (mt/mbql-query categories {:filter [:= $id 1]})- :update_row {(format-field-name :name) "updated_row"})+ :update-row {(format-field-name :name) "updated_row"}):expected {:rows-updated 1}}{:action :model.row/delete:request-body (mt/mbql-query categories {:filter [:= $id 1]}):expected {:rows-deleted 1}}{:action :model.row/update:request-body (assoc (mt/mbql-query categories {:filter [:= $id 10]})- :update_row {(format-field-name :name) "new-category-name"})+ :update-row {(format-field-name :name) "new-category-name"}):expected {:rows-updated 1}}])(deftest feature-flags-test@@ -168,7 +168,7 @@test-scope[{:database db-id:table-id table-id- :values {:name "Toucannery"}}]))))))+ :values {"name" "Toucannery"}}]))))))(defn- row-action? [action](= (namespace action) "row"))@@ -190,9 +190,9 @@(mt/with-actions-enabled(binding [*current-user-permissions-set* (delay #{"/"})](let [query-that-returns-more-than-one (assoc (mt/mbql-query users {:filter [:>= $id 1]})- :update_row {(format-field-name :name) "new-name"})+ :update-row {(format-field-name :name) "new-name"})query-that-returns-zero-row (assoc (mt/mbql-query users {:filter [:= $id Integer/MAX_VALUE]})- :update_row {(format-field-name :name) "new-name"})+ :update-row {(format-field-name :name) "new-name"})result-count (count (mt/rows (qp/process-query query-that-returns-more-than-one)))](is (< 1 result-count))(is (thrown-with-msg? Exception #"Sorry, this would update [\d|,]+ rows, but you can only act on 1"@@ -206,10 +206,8 @@(mt/test-drivers (mt/normal-drivers-with-feature :actions)(mt/with-actions-enabled(binding [*current-user-permissions-set* (delay #{"/"})]- (let [query-that-returns-more-than-one (assoc (mt/mbql-query checkins {:filter [:>= $id 1]})- :update_row {(format-field-name :name) "new-name"})- query-that-returns-zero-row (assoc (mt/mbql-query checkins {:filter [:= $id Integer/MAX_VALUE]})- :update_row {(format-field-name :name) "new-name"})+ (let [query-that-returns-more-than-one (mt/mbql-query checkins {:filter [:>= $id 1]})+ query-that-returns-zero-row (mt/mbql-query checkins {:filter [:= $id Integer/MAX_VALUE]})result-count (count (mt/rows (qp/process-query query-that-returns-more-than-one)))](is (< 1 result-count))(is (thrown-with-msg? Exception #"Sorry, this would delete [\d|,]+ rows, but you can only act on 1"@@ -702,7 +700,7 @@(actions/perform-action!:model.row/update(assoc (mt/mbql-query ants {:filter [:= $id "d6b02fa2-bf7b-4b32-80d5-060b649c9859"]})- :update_row {(format-field-name :name) "updated_row"})))+ :update-row {(format-field-name :name) "updated_row"})))"Update should return the right shape")(is (= "updated_row"(-> (mt/rows (mt/run-mbql-query antsdiff --git a/test/metabase/actions/models_test.clj b/test/metabase/actions/models_test.cljindex 91f5eb7344a..8e3625b4b27 100644--- a/test/metabase/actions/models_test.clj+++ b/test/metabase/actions/models_test.clj@@ -5,8 +5,10 @@[clojure.set :as set][clojure.test :refer :all][metabase.actions.models :as action]+ [metabase.actions.schema :as actions.schema][metabase.driver :as driver][metabase.driver.mysql :as mysql]+ [metabase.lib.core :as lib][metabase.query-processor.preprocess :as qp.preprocess][metabase.sync.core :as sync][metabase.test :as mt]@@ -196,7 +198,7 @@(testing "Update action"(let [action (action/select-action :id action-id)];; Update columns on both the action and the subtype table- (action/update! (assoc action :name "New name" :kind "row/update") action)+ (action/update! (assoc action :name "New name" :kind :row/update) action)(let [new-action (action/select-action :id action-id)](is (partial= {:name "New name":kind :row/update} new-action)))))))))@@ -205,17 +207,19 @@(mt/with-actions-enabled(mt/with-actions [{model-id :id, model-db-id :database_id} {:type :model, :dataset_query (mt/mbql-query categories)}](testing "insert! derives :database_id from the query's database"- (let [action-id (action/insert! {:type :query- :name "derive db insert"- :model_id model-id- :database_id Integer/MAX_VALUE ; bogus; the query targets the model DB- :dataset_query (mt/native-query {:query "update categories set name = 'x' where id = 1"})})]+ (let [action-id (action/insert! (lib/normalize ::actions.schema/action.for-insert+ {:type :query+ :name "derive db insert"+ :model_id model-id+ :database_id Integer/MAX_VALUE ; bogus; the query targets the model DB+ :dataset_query (mt/native-query {:query "update categories set name = 'x' where id = 1"})}))](is (= model-db-id (:database_id (action/select-action :id action-id))))))(testing "update! re-derives :database_id from the query"- (let [action-id (action/insert! {:type :query- :name "derive db update"- :model_id model-id- :dataset_query (mt/native-query {:query "update categories set name = 'x' where id = 1"})})+ (let [action-id (action/insert! (lib/normalize ::actions.schema/action.for-insert+ {:type :query+ :name "derive db update"+ :model_id model-id+ :dataset_query (mt/native-query {:query "update categories set name = 'x' where id = 1"})}))existing (action/select-action :id action-id)];; a :database_id-only update can't repoint the action away from the query's database(action/update! {:id action-id, :database_id Integer/MAX_VALUE} existing)@@ -339,7 +343,7 @@(mt/with-actions-enabled(mt/with-actions [{model-id :id} {:type :model, :dataset_query (mt/mbql-query foo)}](let [action-data {:type :implicit- :kind "row/create"+ :kind :row/create:name "create foo":model_id model-id}action-id (action/insert! action-data)@@ -360,7 +364,7 @@(mt/with-actions-enabled(mt/with-actions [{model-id :id} {:type :model, :dataset_query (mt/mbql-query foo)}](let [action-data {:type :implicit- :kind "row/create"+ :kind :row/create:name "create foo":model_id model-id}action-id (action/insert! action-data)diff --git a/test/metabase/actions/test_util.clj b/test/metabase/actions/test_util.cljindex c306e88cff9..fa6d7f21666 100644--- a/test/metabase/actions/test_util.clj+++ b/test/metabase/actions/test_util.clj@@ -6,11 +6,13 @@[clojure.test :refer :all][java-time.api :as t][metabase.actions.models :as action]+ [metabase.actions.schema :as actions.schema][metabase.driver :as driver][metabase.driver.ddl.interface :as ddl.i][metabase.driver.sql-jdbc.connection :as sql-jdbc.conn][metabase.driver.sql.query-processor :as sql.qp][metabase.driver.util :as driver.u]+ [metabase.lib.core :as lib][metabase.query-processor.test-util :as qp.test-util][metabase.test.data :as data][metabase.test.data.dataset-definitions :as defs]@@ -175,6 +177,11 @@(is (= [[74]](row-count))))))))))+(defn- insert-action!+ "Normalizes `action` the way the REST API decodes one and inserts it, returning its id."+ [action]+ (action/insert! (lib/normalize ::actions.schema/action.for-insert action)))+(defmulti ^:private create-action*!{:arglists '([options-map model-id])}(fn [options-map _model-id]@@ -182,7 +189,7 @@(defmethod create-action*! :query[options-map model-id]- (let [action-id (action/insert!+ (let [action-id (insert-action!(merge {:model_id model-id:name "Query Example":parameters [{:id "id"@@ -217,7 +224,7 @@(defmethod create-action*! :implicit[options-map model-id]- (let [action-id (action/insert! (merge+ (let [action-id (insert-action! (merge{:type :implicit:name "Update Example":kind "row/update"@@ -230,7 +237,7 @@(defmethod create-action*! :http[options-map model-id]- (let [action-id (action/insert! (merge+ (let [action-id (insert-action! (merge{:type :http:name "Echo Example":template {:url (client/build-url "testing/echo[[?fail={{fail}}]]" {})diff --git a/test/metabase/analytics/metaplow_test.clj b/test/metabase/analytics/metaplow_test.cljindex 317bfab1e35..66f12b2cf98 100644--- a/test/metabase/analytics/metaplow_test.clj+++ b/test/metabase/analytics/metaplow_test.clj@@ -95,10 +95,10 @@(swap! received conj payload)(.countDown latch):sent)]- (doseq [i (range 200)]+ (doseq [i (range 1 201)](metaplow/track-event! :snowplow/dashboard {:event :dashboard-created :dashboard-id i}))(is (true? (.await latch 1 TimeUnit/SECONDS))"Pipeline did not process all 200 events within 1s")(is (= 200 (count @received)))- (is (= (set (range 200))+ (is (= (set (range 1 201))(set (map #(get-in % [:payload :data "dashboard_id"]) @received)))))))))diff --git a/test/metabase/analyze/classifiers/data_sensitivity_test.clj b/test/metabase/analyze/classifiers/data_sensitivity_test.cljindex 163ddd617ca..1ca3a305f8c 100644--- a/test/metabase/analyze/classifiers/data_sensitivity_test.clj+++ b/test/metabase/analyze/classifiers/data_sensitivity_test.clj@@ -9,7 +9,7 @@(infer field-name base-type nil nil))([field-name base-type semantic-type table-context](classifiers.data-sensitivity/infer-data-sensitivity- {:name field-name :base_type base-type :semantic_type semantic-type}+ {:id 1 :name field-name :base_type base-type :semantic_type semantic-type}table-context)))(deftest ^:parallel name->tokens-test@@ -230,7 +230,8 @@(deftest ^:parallel fingerprint-rule-test(let [infer-fp (fn [base-type percent-email](classifiers.data-sensitivity/infer-data-sensitivity- {:name "col_7"+ {:id 1+ :name "col_7":base_type base-type:fingerprint {:type {:type/Text {:percent-email percent-email}}}}nil))]diff --git a/test/metabase/analyze/query_results_test.clj b/test/metabase/analyze/query_results_test.cljindex 7e60858280b..d450f339c54 100644--- a/test/metabase/analyze/query_results_test.clj+++ b/test/metabase/analyze/query_results_test.clj@@ -38,7 +38,7 @@"Return the desired storage format for the column metadata coming back from `results` and fingerprint the `results`."[{:keys [rows], :as result}]{:pre [(map? result) (:cols result)]}- (add-insights rows result))+ (add-insights rows (select-keys result [:cols])))(defn- query->result-metadata[query-map]diff --git a/test/metabase/api/macros/defendpoint/closed_schemas_test.clj b/test/metabase/api/macros/defendpoint/closed_schemas_test.cljindex eb0c05aa6d4..db876f34a8a 100644--- a/test/metabase/api/macros/defendpoint/closed_schemas_test.clj+++ b/test/metabase/api/macros/defendpoint/closed_schemas_test.clj@@ -4,6 +4,7 @@[malli.core :as mc][metabase.api.macros.defendpoint.closed-schemas :as closed-schemas][metabase.lib.schema.common :as lib.schema.common]+ [metabase.util.malli.closed-schemas :as mu.closed-schemas][metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms]))@@ -58,12 +59,15 @@::nested[:map {:closed true} [:a ::open]][:or :int ::open]+ map?+ [:map {:closed true} [:settings map?]][:multi {:dispatch :type} [:a [:map [:type [:= :a]]]]]))(deftest ^:parallel keyword-keyed-map-of-is-found-test(are [schema] (= [:keyword-keyed-map-of] (kinds schema))[:map-of :keyword :int][:map-of [:keyword {:description "a key"}] :int]+ [:map-of [:or :keyword :string] :int][:map {:closed true} [:bag [:map-of :keyword :string]]])(testing "a `:some` key is both an open bag of keys and a value of any type"(is (= [:keyword-keyed-map-of :any] (kinds [:map-of :some :string])))))@@ -73,10 +77,8 @@:anyany?:some- map?[:maybe :any][:map {:closed true} [:value :any]]- [:map {:closed true} [:settings map?]][:map-of :string :any](ms/string-keyed-map :any)))@@ -134,9 +136,9 @@(deftest check-throws-with-the-offending-schemas-test(testing "off in prod, and off when disabled"- (binding [closed-schemas/*enabled* false]+ (binding [mu.closed-schemas/*enabled* false](is (nil? (closed-schemas/check! :body [:map [:a :int]])))))- (binding [closed-schemas/*enabled* true]+ (binding [mu.closed-schemas/*enabled* true](is (nil? (closed-schemas/check! :body [:map {:closed true} [:a :int]])))(testing "response schemas are not checked: only a request is stripped"(is (nil? (closed-schemas/check! :response [:map [:a :int]]))))diff --git a/test/metabase/api/macros/defendpoint/open_api_test.clj b/test/metabase/api/macros/defendpoint/open_api_test.cljindex 849f96398d5..cdeb31ac26e 100644--- a/test/metabase/api/macros/defendpoint/open_api_test.clj+++ b/test/metabase/api/macros/defendpoint/open_api_test.clj@@ -38,7 +38,8 @@:required ["id"]:properties {"id" {:type :integer}}}}(#'defendpoint.open-api/fix-json-schema- (mjs/transform (ms/maps-with-unique-key [:sequential [:map [:id :int]]] :id)))))))+ (#'defendpoint.open-api/normalize-raw-json-schema+ (mjs/transform (ms/maps-with-unique-key [:sequential [:map [:id :int]]] :id))))))))(deftest ^:parallel default-params-are-optional-test(testing "a param carrying a :default is advertised as optional, since omitting it is safe"@@ -60,7 +61,7 @@(deftest ^:parallel collect-definitions-test(binding [defendpoint.open-api/*definitions* (atom [])]- (is (=? {:properties {:value {:$ref "#/components/schemas/metabase.lib.schema.common.non-blank-string"}}}+ (is (=? {:properties {"value" {:$ref "#/components/schemas/metabase.lib.schema.common.non-blank-string"}}}(#'defendpoint.open-api/mjs-collect-definitions [:map [:value ::lib.schema.common/non-blank-string]])))(is (= [{"metabase.lib.schema.common.non-blank-string" {:type :string, :minLength 1}}]@@#'defendpoint.open-api/*definitions*))))diff --git a/test/metabase/api/macros_test.clj b/test/metabase/api/macros_test.cljindex 41f7cc7b85c..bed11aba410 100644--- a/test/metabase/api/macros_test.clj+++ b/test/metabase/api/macros_test.clj@@ -82,24 +82,24 @@[{:keys [id]} :- [:map[:id pos-int?]]](neat))- {:route {:path "/move/:id", :regexes {:id #"[0-9]+"}}}+ {:route {:path "/move/:id", :regexes {"id" #"[0-9]+"}}}'(:post "/move/:id"[{:keys [id]} :- [:map[:id uuid?]]](neat))- {:route {:path "/move/:id", :regexes {:id #"[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}"}}}+ {:route {:path "/move/:id", :regexes {"id" #"[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}"}}}'(:post "/move/:id"[{:keys [id]} :- [:map[:id ::id]]](neat))- {:route {:path "/move/:id", :regexes {:id #"[0-9]+"}}}+ {:route {:path "/move/:id", :regexes {"id" #"[0-9]+"}}}'(:post "/move/:id"[{:keys [id]} :- RouteParams](neat))- {:route {:path "/move/:id", :regexes {:id #"[abc]{4}"}}}))+ {:route {:path "/move/:id", :regexes {"id" #"[abc]{4}"}}}))(deftest ^:parallel multipart-tempfile-cleanup-on-throw-test(testing "tempfiles are deleted when the handler throws, e.g. on a param validation 400"diff --git a/test/metabase/batch_processing/impl_test.clj b/test/metabase/batch_processing/impl_test.cljindex d5749aa175b..d8e9c5be85f 100644--- a/test/metabase/batch_processing/impl_test.clj+++ b/test/metabase/batch_processing/impl_test.clj@@ -15,8 +15,8 @@:capacity 5:interval (* 10 1000))](u/with-timeout 1000- (grouper/submit! g 1))- (is (= [1] @processed?))))))+ (grouper/submit! g {:id 1}))+ (is (= [{:id 1}] @processed?))))))(deftest flush-test(testing "flush! processes everything already submitted, without exposing its sentinel to the batch fn"@@ -27,15 +27,15 @@:capacity 5:interval (* 60 1000))](try- (grouper/submit! g 1)- (grouper/submit! g 2)+ (grouper/submit! g {:id 1})+ (grouper/submit! g {:id 2})(is (= [] @batches))(u/with-timeout 5000(grouper/flush! g))- (is (= [[1 2]] @batches))+ (is (= [[{:id 1} {:id 2}]] @batches))(testing "a flush with nothing queued does not call the batch fn"(u/with-timeout 5000(grouper/flush! g))- (is (= [[1 2]] @batches)))+ (is (= [[{:id 1} {:id 2}]] @batches)))(finally(grouper/shutdown! g))))))diff --git a/test/metabase/bookmarks/models/bookmark_test.clj b/test/metabase/bookmarks/models/bookmark_test.cljindex 850bc0824e8..97f8160ea2e 100644--- a/test/metabase/bookmarks/models/bookmark_test.clj+++ b/test/metabase/bookmarks/models/bookmark_test.clj@@ -39,9 +39,16 @@:report_dashboard.archived false:collection.description "Collection description":collection.archived true+ :collection.authority_level nil+ :collection.is_remote_synced nil:report_card.card_type nil:created_at #t "2022-09-14T17:45:13.444716Z"- :collection.name "Test Collection"}]+ :collection.name "Test Collection"+ :document.name nil+ :document.archived nil+ :exploration.name nil+ :exploration.description nil+ :exploration.archived nil}](is (= {:item_id 853:name "Test Dashboard":type "dashboard"diff --git a/test/metabase/channel/email/result_attachment_test.clj b/test/metabase/channel/email/result_attachment_test.cljindex 3c385ac1dcf..706b334d1a3 100644--- a/test/metabase/channel/email/result_attachment_test.clj+++ b/test/metabase/channel/email/result_attachment_test.clj@@ -28,7 +28,7 @@(testing "CSV email attachments start with a UTF-8 BOM so they open correctly in Excel (#75875)"(let [results {:database_id (mt/id):row_count 1- :data {:cols [{:name "n" :base_type :type/Integer}]+ :data {:cols [{:name "n" :display_name "n" :base_type :type/Integer}]:rows [[1]]:viz-settings {}}}stream! (fn [opts]diff --git a/test/metabase/channel/impl/email_test.clj b/test/metabase/channel/impl/email_test.cljindex ce16e2bbeac..30efb1fceb1 100644--- a/test/metabase/channel/impl/email_test.clj+++ b/test/metabase/channel/impl/email_test.clj@@ -2,6 +2,7 @@(:require[clojure.string :as str][clojure.test :refer :all]+ [java-time.api :as t][metabase.channel.core :as channel][metabase.channel.email :as email][metabase.channel.impl.email :as email.impl]@@ -85,6 +86,29 @@(is (true? (-> result first :card :format_rows))"Should include format_rows setting using the fallback match")))))+(defn- test-dashcard+ "A DashboardCard row for card `card-id` on dashboard `dashboard-id`, as a Dashboard Subscription part carries it."+ [id dashboard-id card-id]+ {:id id+ :dashboard_id dashboard-id+ :card_id card-id+ :created_at (t/offset-date-time)+ :updated_at (t/offset-date-time)+ :size_x 4+ :size_y 4+ :row 0+ :col 0+ :parameter_mappings []+ :visualization_settings {}+ :entity_id "test-dashcard-entity"+ :action_id nil+ :dashboard_tab_id nil+ :inline_parameters nil})++(def ^:private empty-result+ "A QP result with no rows."+ {:data {:cols [] :rows []} :row_count 0})+(deftest dashboard-notification-dashcard-links-test(testing "Dashboard notification renders dashcard links with parameters like Slack implementation"(let [dashboard-id 42@@ -103,8 +127,8 @@:dashboard_parts [{:type :card:card {:id card-id:name "Test Card"}- :dashcard {:id dashcard-id- :dashboard_id dashboard-id}}]+ :dashcard (test-dashcard dashcard-id dashboard-id card-id)+ :result empty-result}]:dashboard_subscription {:id 789:dashboard_subscription_dashcards []}}}recipients [{:type :notification-recipient/userdiff --git a/test/metabase/channel/impl/slack_test.clj b/test/metabase/channel/impl/slack_test.cljindex d90b50e4bfb..1d4aa21f762 100644--- a/test/metabase/channel/impl/slack_test.clj+++ b/test/metabase/channel/impl/slack_test.clj@@ -2,6 +2,7 @@(:require[clojure.string :as str][clojure.test :refer :all]+ [java-time.api :as t][metabase.channel.core :as channel][metabase.channel.impl.slack :as channel.slack][metabase.channel.slack :as slack]@@ -154,6 +155,29 @@(let [links (render-dashboard-links false)](is (= 2 (count links)))))))+(defn- test-dashcard+ "A DashboardCard row for card `card-id` on dashboard `dashboard-id`, as a Dashboard Subscription part carries it."+ [id dashboard-id card-id]+ {:id id+ :dashboard_id dashboard-id+ :card_id card-id+ :created_at (t/offset-date-time)+ :updated_at (t/offset-date-time)+ :size_x 4+ :size_y 4+ :row 0+ :col 0+ :parameter_mappings []+ :visualization_settings {}+ :entity_id "test-dashcard-entity"+ :action_id nil+ :dashboard_tab_id nil+ :inline_parameters nil})++(def ^:private empty-result+ "A QP result with no rows."+ {:data {:cols [] :rows []} :row_count 0})+(deftest dashboard-card-links-include-parameters-test(let [dashboard-id 42card-id 123@@ -168,7 +192,8 @@:parameters dashboard-params:dashboard_parts [{:type :card:card {:id card-id :name "Test Card"}- :dashcard {:id 456 :dashboard_id dashboard-id}}]}+ :dashcard (test-dashcard 456 dashboard-id card-id)+ :result empty-result}]}:creator {:common_name "Test User"}}recipient {:type :notification-recipient/raw-value:details {:value "#test-channel"}}]@@ -187,8 +212,8 @@notification {:payload_type :notification/dashboard:payload {:dashboard {:id 1 :name "Test Dashboard"}:parameters []- :dashboard_parts [{:type :card :card {:id 1 :name "Good Card"} :dashcard {:id 10 :dashboard_id 1}}- {:type :card :card {:id 2 :name "Bad Card"} :dashcard {:id 20 :dashboard_id 1}}]}+ :dashboard_parts [{:type :card :card {:id 1 :name "Good Card"} :dashcard (test-dashcard 10 1 1) :result empty-result}+ {:type :card :card {:id 2 :name "Bad Card"} :dashcard (test-dashcard 20 1 2) :result empty-result}]}:creator {:common_name "Test User"}}recipient {:type :notification-recipient/raw-value :details {:value "#test-channel"}}](mt/with-dynamic-fn-redefs [slack/upload-file! (fn [_ _] {:id "uploaded-file-id"})diff --git a/test/metabase/channel/render/body_test.clj b/test/metabase/channel/render/body_test.cljindex 9a092150dca..627a73e7b33 100644--- a/test/metabase/channel/render/body_test.clj+++ b/test/metabase/channel/render/body_test.clj@@ -1421,8 +1421,10 @@(def ^:private pivot-test-data"A `:pivot` query result: rows R, cols C, single measure m, plus the pivot-grouping column the QP emits.`:pivot-export-options` carries the row/col/measure column indexes in the pivot-grouping-free space (R=0, C=1, m=2)."- {:cols [{:name "R" :base_type :type/Text} {:name "C" :base_type :type/Text}- {:name "pivot-grouping" :base_type :type/Integer} {:name "m" :base_type :type/Integer}]+ {:cols [{:name "R" :display_name "R" :base_type :type/Text}+ {:name "C" :display_name "C" :base_type :type/Text}+ {:name "pivot-grouping" :display_name "pivot-grouping" :base_type :type/Integer}+ {:name "m" :display_name "m" :base_type :type/Integer}]:rows [["a" "x" 0 10] ["a" "y" 0 20] ["b" "x" 0 30] ["b" "y" 0 40]]:format-rows? true:pivot-export-options {:pivot-rows [0] :pivot-cols [1] :pivot-measures [2]}})@@ -1447,7 +1449,8 @@(testing "a pivot card with no column split degrades to a flat table without erroring"(let [part (body/render :pivot :inline "UTC"{:display :pivot :visualization_settings {}} nil- {:cols [{:name "a" :base_type :type/Text} {:name "b" :base_type :type/Number}]+ {:cols [{:name "a" :display_name "a" :base_type :type/Text}+ {:name "b" :display_name "b" :base_type :type/Number}]:rows [["x" 1]]})](is (some? (:content part)))))(testing "the card's conditional formatting colors the measure value cells"@@ -1492,9 +1495,11 @@;; uncolored) and the m1 cells labeled m2 (11-44, not > 50 -> uncolored) -- so the m2 cells being the colored;; ones confirms the measure mapping. Totals are off so only the data cells remain.(let [split {:rows ["R"] :columns ["C"] :values ["m1" "m2"]}- cols [{:name "R" :base_type :type/Text} {:name "C" :base_type :type/Text}- {:name "pivot-grouping" :base_type :type/Integer}- {:name "m1" :base_type :type/Integer} {:name "m2" :base_type :type/Integer}]+ cols [{:name "R" :display_name "R" :base_type :type/Text}+ {:name "C" :display_name "C" :base_type :type/Text}+ {:name "pivot-grouping" :display_name "pivot-grouping" :base_type :type/Integer}+ {:name "m1" :display_name "m1" :base_type :type/Integer}+ {:name "m2" :display_name "m2" :base_type :type/Integer}]data {:cols cols:rows [["a" "x" 0 11 100] ["a" "y" 0 22 200]["b" "x" 0 33 300] ["b" "y" 0 44 400]]diff --git a/test/metabase/channel/render/pdf_test.clj b/test/metabase/channel/render/pdf_test.cljindex e4e2865b530..b5750482369 100644--- a/test/metabase/channel/render/pdf_test.clj+++ b/test/metabase/channel/render/pdf_test.clj@@ -824,10 +824,10 @@assembled {:card {:display :pivot:visualization_settings {:pivot_table.column_split {:rows ["R"] :columns ["C"] :values ["m"]}}}:dashcard nil- :result {:data {:cols [{:name "R" :base_type :type/Text}- {:name "C" :base_type :type/Text}- {:name "pivot-grouping" :base_type :type/Integer}- {:name "m" :base_type :type/Integer}]+ :result {:data {:cols [{:name "R" :display_name "R" :base_type :type/Text}+ {:name "C" :display_name "C" :base_type :type/Text}+ {:name "pivot-grouping" :display_name "pivot-grouping" :base_type :type/Integer}+ {:name "m" :display_name "m" :base_type :type/Integer}]:rows [["a" "x" 0 10]["a" "y" 0 20]["b" "x" 0 30]diff --git a/test/metabase/channel/render/table_test.clj b/test/metabase/channel/render/table_test.cljindex 8f64e967900..a776498aceb 100644--- a/test/metabase/channel/render/table_test.clj+++ b/test/metabase/channel/render/table_test.clj@@ -277,7 +277,7 @@(let [data {:cols [{:name "A" :display_name "A" :base_type :type/Integer :semantic_type nil}]:rows [[5] [10]]:format-rows? true- :results_metadata {:columns [{:name "A" :base_type :type/Integer+ :results_metadata {:columns [{:name "A" :display_name "A" :base_type :type/Integer:fingerprint {:global {:distinct-count 2}}}]}:viz-settings {::mb.viz/column-settings{{::mb.viz/column-name "A"} {::mb.viz/show-mini-bar true}}}}@@ -295,8 +295,10 @@(is (seq (hik.s/select (hik.s/find-in-text #"^5$") doc)))(is (seq (hik.s/select (hik.s/find-in-text #"^10$") doc)))))))-(defn- render-table [dashcard results]- (channel.render/render-pulse-card :attachment "America/Los_Angeles" render.tu/test-card dashcard results))+(defn- render-table [viz-settings results]+ (channel.render/render-pulse-card :attachment "America/Los_Angeles"+ (update render.tu/test-card :visualization_settings merge viz-settings)+ nil results))(deftest attachment-rows-limit-test(doseq [[test-explanation env-var-value expected]@@ -309,8 +311,7 @@(mt/with-temp-env-var-value! ["MB_ATTACHMENT_TABLE_ROW_LIMIT" env-var-value](is (= expected(count (-> (render-table- {:visualization_settings {:table.columns- [{:name "a" :enabled true}]}}+ {:table.columns [{:name "a" :enabled true}]}{:data {:cols [{:name "a",:display_name "a",:base_type :type/BigIntegerdiff --git a/test/metabase/dashboards/models/dashboard_card_test.clj b/test/metabase/dashboards/models/dashboard_card_test.cljindex b7e20e9b90b..67348e63262 100644--- a/test/metabase/dashboards/models/dashboard_card_test.clj+++ b/test/metabase/dashboards/models/dashboard_card_test.clj@@ -179,7 +179,6 @@(testing "return value from the update call should be nil"(is (nil? (dashboard-card/update-dashboard-card!{:id dashcard-id- :actor_id (mt/user->id :rasta):dashboard_id nil:card_id nil:size_x 5@@ -231,21 +230,21 @@(testing "Should have more calls if there are changes to the dashcards"(t2/with-call-count [call-count](dashboard/update-dashcards! dashboard [{:id (:id dashcard-1)- :cardId card-id+ :card_id card-id:row 1:col 2:size_x 3:size_y 4:series [{:id series-id-1}]}{:id (:id dashcard-2)- :cardId card-id+ :card_id card-id:row 1:col 2:size_x 3:size_y 4:series [{:id series-id-2}]}{:id (:id dashcard-3)- :cardId card-id+ :card_id card-id:row 1:col 2:size_x 3diff --git a/test/metabase/dashboards_rest/api_test.clj b/test/metabase/dashboards_rest/api_test.cljindex 7edf23b2485..01bd79570e6 100644--- a/test/metabase/dashboards_rest/api_test.clj+++ b/test/metabase/dashboards_rest/api_test.clj@@ -1657,10 +1657,10 @@(deftest cards-to-copy-test(testing "Identifies all cards to be copied"- (let [dashcards [{:card_id 1 :card (card-model {:id 1}) :series [(card-model {:id 2})]}- {:card_id 3 :card (card-model {:id 3})}+ (let [dashcards [{:id 11, :card_id 1 :card (card-model {:id 1}) :series [(card-model {:id 2})]}+ {:id 13, :card_id 3 :card (card-model {:id 3})};; this guy does not even reach the discard pile- {:action_id 123}]]+ {:id 14, :action_id 123}]](binding [*readable-card-ids* #{1 2 3}](is (= {:copy {1 {:id 1} 2 {:id 2} 3 {:id 3}}:reference {}@@ -1668,24 +1668,24 @@(#'api.dashboard/cards-to-copy true dashcards))))))(testing "Identifies cards which cannot be copied"(testing "If they are in a series"- (let [dashcards [{:card_id 1 :card (card-model {:id 1}) :series [(card-model {:id 2})]}- {:card_id 3 :card (card-model {:id 3})}]]+ (let [dashcards [{:id 11, :card_id 1 :card (card-model {:id 1}) :series [(card-model {:id 2})]}+ {:id 13, :card_id 3 :card (card-model {:id 3})}]](binding [*readable-card-ids* #{1 3}](is (= {:copy {1 {:id 1} 3 {:id 3}}:reference {}:discard [{:id 2}]}(#'api.dashboard/cards-to-copy true dashcards))))))(testing "When the base of a series lacks permissions"- (let [dashcards [{:card_id 1 :card (card-model {:id 1}) :series [(card-model {:id 2})]}- {:card_id 3 :card (card-model {:id 3})}]]+ (let [dashcards [{:id 11, :card_id 1 :card (card-model {:id 1}) :series [(card-model {:id 2})]}+ {:id 13, :card_id 3 :card (card-model {:id 3})}]](binding [*readable-card-ids* #{3}](is (= {:copy {3 {:id 3}}:reference {}:discard [{:id 1} {:id 2}]}(#'api.dashboard/cards-to-copy true dashcards)))))))(testing "Identifies cards to be referenced"- (let [dashcards [{:card_id 1 :card (card-model {:id 1}) :series [(card-model {:id 2})]}- {:card_id 3 :card (card-model {:id 3})}]]+ (let [dashcards [{:id 11, :card_id 1 :card (card-model {:id 1}) :series [(card-model {:id 2})]}+ {:id 13, :card_id 3 :card (card-model {:id 3})}]](binding [*readable-card-ids* #{1 2 3}](is (= {:reference {1 {:id 1}2 {:id 2}@@ -1694,8 +1694,8 @@:discard []}(#'api.dashboard/cards-to-copy false dashcards))))))(testing "Identifies cards that cannot be referenced"- (let [dashcards [{:card_id 1 :card (card-model {:id 1}) :series [(card-model {:id 2})]}- {:card_id 3 :card (card-model {:id 3})}]]+ (let [dashcards [{:id 11, :card_id 1 :card (card-model {:id 1}) :series [(card-model {:id 2})]}+ {:id 13, :card_id 3 :card (card-model {:id 3})}]](binding [*readable-card-ids* #{1 3}](is (= {:reference {1 {:id 1}3 {:id 3}}@@ -3095,7 +3095,6 @@:table_id (mt/id :orders):dataset_query (mt/mbql-query orders)}:model/Card {card-id :id} {:database_id (mt/id)- :table_id (str "card__" saved-query-id):dataset_query {:database (mt/id):type :query:query {:source-table (str "card__" saved-query-id)diff --git a/test/metabase/driver/common_test.clj b/test/metabase/driver/common_test.cljindex ec2726c3738..fe65d0c40d3 100644--- a/test/metabase/driver/common_test.clj+++ b/test/metabase/driver/common_test.clj@@ -76,7 +76,8 @@(are [details expected] (= expected(driver.common/json-unfolding-default{:lib/type :metadata/database- :details details}))+ :id 1+ :details details})){} true{:json-unfolding nil} true{:json-unfolding true} truediff --git a/test/metabase/driver/connection_test.clj b/test/metabase/driver/connection_test.cljindex d4264ca3026..5cb38e936e8 100644--- a/test/metabase/driver/connection_test.clj+++ b/test/metabase/driver/connection_test.clj@@ -14,6 +14,7 @@(let [details {:host "read-host" :port 5432}write-data-details {:host "write-host" :port 5432}database {:lib/type :metadata/database+ :id 1:details details:write-data-details write-data-details}](is (=? details@@ -27,6 +28,7 @@(testing "effective-details returns write_data_details when :write-data and details exist"(testing "with kebab-case key (Lib metadata style)"(let [database {:lib/type :metadata/database+ :id 1:details details:write-data-details write-data-details}](driver.conn/with-write-connection@@ -46,6 +48,7 @@(let [details {:host "read-host" :port 5432}](testing "effective-details falls back to :details when :write-data but no write details"(let [database {:lib/type :metadata/database+ :id 1:details details:write-data-details nil}](driver.conn/with-write-connection@@ -53,6 +56,7 @@(driver.conn/effective-details database)))))(testing "also when :write_data_details key is missing entirely"(let [database {:lib/type :metadata/database+ :id 1:details details}](driver.conn/with-write-connection(is (=? details@@ -64,6 +68,7 @@(let [details {:host "read-host" :port 5432}write-data-details {:host "write-host" :port 5432}database {:lib/type :metadata/database+ :id 1:details details:write-data-details write-data-details}](driver.conn/with-write-connection@@ -92,6 +97,7 @@(let [details {:host "read-host" :port 5432}admin-details {:host "admin-host" :port 5432 :user "root"}database {:lib/type :metadata/database+ :id 1:details details:admin-details admin-details}](is (=? details@@ -103,6 +109,7 @@(let [details {:host "read-host" :port 5432}admin-details {:host "admin-host" :port 5432 :user "root"}database {:lib/type :metadata/database+ :id 1:details details:admin-details admin-details}](driver.conn/with-admin-connectiondiff --git a/test/metabase/driver/mysql_test.clj b/test/metabase/driver/mysql_test.cljindex 7c1734c5f4f..6781b5a6c5c 100644--- a/test/metabase/driver/mysql_test.clj+++ b/test/metabase/driver/mysql_test.clj@@ -748,7 +748,8 @@(let [field (t2/select-one :model/Field :table_id (u/id table) :name "json_bit → 1234")](mt/with-metadata-provider (mt/id)(let [field-clause [:field- {:binning+ {:lib/uuid (str (random-uuid))+ :binning{:strategy :num-bins,:num-bins 100,:min-value 0.75,diff --git a/test/metabase/driver/postgres_test.clj b/test/metabase/driver/postgres_test.cljindex 37efb71aa0f..feb0884d86c 100644--- a/test/metabase/driver/postgres_test.clj+++ b/test/metabase/driver/postgres_test.clj@@ -472,7 +472,8 @@:nfc-path ["jsons" "values" "qty"]:database-type "integer"})]})(let [field-clause [:field- {:binning+ {:lib/uuid (str (random-uuid))+ :binning{:strategy :num-bins:num-bins 100:min-value 0.75diff --git a/test/metabase/driver/sql/parameters/substitute_test.clj b/test/metabase/driver/sql/parameters/substitute_test.cljindex ac576d31b3a..8fb41473a4e 100644--- a/test/metabase/driver/sql/parameters/substitute_test.clj+++ b/test/metabase/driver/sql/parameters/substitute_test.clj@@ -11,6 +11,7 @@[metabase.legacy-mbql.normalize :as mbql.normalize][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]+ [metabase.lib.schema.metadata :as lib.schema.metadata][metabase.lib.test-metadata :as meta][metabase.lib.test-util :as lib.tu][metabase.query-processor.compile :as qp.compile]@@ -517,15 +518,10 @@(mu/defn- expand**"Expand parameters inside a top-level native `query`. Not recursive. Expands against `meta/metadata-provider` unlessan explicit metadata provider `mp` is supplied (handy for overriding field metadata)."- ([query]+ ([query :- :metabase.lib.util/query-like](expand** meta/metadata-provider query))- ([mp- {:keys [parameters], :as query} :- [:or- [:map- [:lib/type [:= :mbql/query]]]- [:map- [:native [:map- [:query some?]]]]]]+ ([mp :- ::lib.schema.metadata/metadata-providerable+ {:keys [parameters], :as query} :- :metabase.lib.util/query-like](driver/with-driver :h2(-> (if (:lib/type query)(lib/query mp (dissoc query :parameters))diff --git a/test/metabase/driver/sql/parameters/substitution_test.clj b/test/metabase/driver/sql/parameters/substitution_test.cljindex 038bf9e9f8c..36710f71c9b 100644--- a/test/metabase/driver/sql/parameters/substitution_test.clj+++ b/test/metabase/driver/sql/parameters/substitution_test.clj@@ -44,9 +44,10 @@:prepared-statement-args ["Doohickey"]}(#'sql.params.substitution/field-filter->replacement-snippet-info:h2- {:field (lib.metadata/field (qp.store/metadata-provider) (meta/id :venues :name))- :value {:type :string/=- :value ["Doohickey"]}}))))))+ (lib/parsed-field-filter-param+ (lib.metadata/field (qp.store/metadata-provider) (meta/id :venues :name))+ {:type :string/=+ :value ["Doohickey"]})))))))(deftest ^:parallel field-filter-nil-schema-snippet-test(testing "field-filter snippet omits schema when the field's table has no schema (#29786)"@@ -58,9 +59,10 @@:prepared-statement-args ["Doohickey"]}(#'sql.params.substitution/field-filter->replacement-snippet-info:h2- {:field (lib.metadata/field (qp.store/metadata-provider) (meta/id :venues :name))- :value {:type :string/=- :value ["Doohickey"]}})))))))+ (lib/parsed-field-filter-param+ (lib.metadata/field (qp.store/metadata-provider) (meta/id :venues :name))+ {:type :string/=+ :value ["Doohickey"]}))))))))(deftest ^:parallel field-filter->replacement-snippet-info-test-2(testing "Compound filters should be wrapped in parens"diff --git a/test/metabase/driver/sql/query_processor_test.clj b/test/metabase/driver/sql/query_processor_test.cljindex dd0125af6ca..d058aaf3f25 100644--- a/test/metabase/driver/sql/query_processor_test.clj+++ b/test/metabase/driver/sql/query_processor_test.clj@@ -684,9 +684,7 @@(-> (lib.tu.macros/mbql-query venues{:source-table $$venues:joins [{:alias "cat"- :source-query {:source-table $$categories- ;; this will prevent the nesting from getting collapsed- ::whatever true}+ :source-query {:source-table $$categories}:condition [:= $category-id &cat.*categories.id]}]:order-by [[:asc $name]]:limit 3})diff --git a/test/metabase/driver/sql_jdbc/connection_test.clj b/test/metabase/driver/sql_jdbc/connection_test.cljindex 6ad94ea2380..6b7dd463f67 100644--- a/test/metabase/driver/sql_jdbc/connection_test.clj+++ b/test/metabase/driver/sql_jdbc/connection_test.clj@@ -342,6 +342,7 @@:value (.getBytes "super secret"):creator_id (mt/user->id :crowberto)}](let [db {:lib/type :metadata/database+ :id 1:engine :postgres:details {:ssl true:ssl-mode "verify-ca"diff --git a/test/metabase/driver/sql_jdbc_test.clj b/test/metabase/driver/sql_jdbc_test.cljindex ceeae671932..de232b92aaf 100644--- a/test/metabase/driver/sql_jdbc_test.clj+++ b/test/metabase/driver/sql_jdbc_test.clj@@ -290,7 +290,7 @@%))](sql.qp/->honeysqldriver/*driver*- [:= {} col-ref [:value {:base-type :type/UUID} (str uuid)]])))+ [:= {} col-ref (lib/normalize [:value {:base-type :type/UUID} (str uuid)])])))(is (=? [:= [:metabase.util.honey-sql-2/identifier :field [field]](some-fn #(= uuid %)#(= [:metabase.util.honey-sql-2/typeddiff --git a/test/metabase/driver/sync_test.clj b/test/metabase/driver/sync_test.cljindex 69f7409fc0c..6b8dee5d9fb 100644--- a/test/metabase/driver/sync_test.clj+++ b/test/metabase/driver/sync_test.clj@@ -35,12 +35,14 @@(testing "Schema filter function should not NPE if you pass in a `nil` schema (#38156)"(testing "inclusion -- don't include nil schemas"(let [db {:lib/type :metadata/database+ :id 1:details {:schema-filters-type "inclusion":schema-filters-patterns "x"}:engine :postgres}](is (not (driver.s/include-schema? db nil)))))(testing "exclusion -- don't exclude nil schemas"(let [db {:lib/type :metadata/database+ :id 1:details {:schema-filters-type "exclusion":schema-filters-patterns "x"}:engine :postgres}]diff --git a/test/metabase/driver/util_test.clj b/test/metabase/driver/util_test.cljindex 0890c5a4e96..40428906be0 100644--- a/test/metabase/driver/util_test.clj+++ b/test/metabase/driver/util_test.clj@@ -100,14 +100,14 @@:required false}{:name "ssl"}{:name "use-keystore"- :visible-if {:ssl true}}+ :visible-if {"ssl" true}}{:name "keystore-password-value":display-name "Keystore Password",:type "password",:required false,- :visible-if {:use-keystore true+ :visible-if {"use-keystore" true;; this should have been filled in as a transitive dependency- :ssl true}}+ "ssl" true}}{:name "keystore-options":display-name "Keystore":options [{:name "Local file path"@@ -116,17 +116,17 @@:value "uploaded"}]:type "select":default "local"- :visible-if {:use-keystore true- :ssl true}}+ :visible-if {"use-keystore" true+ "ssl" true}}{:name "keystore-value":type "textFile":treat-before-posting "base64"- :visible-if {:keystore-options "uploaded"}}+ :visible-if {"keystore-options" "uploaded"}}{:name "keystore-path":type "string"- :visible-if {:keystore-options "local"- :use-keystore true- :ssl true}}]+ :visible-if {"keystore-options" "local"+ "use-keystore" true+ "ssl" true}}]false][[{:name "host"}{:name "password-value"@@ -135,16 +135,16 @@:required false}{:name "ssl"}{:name "use-keystore"- :visible-if {:ssl true}}+ :visible-if {"ssl" true}}{:name "keystore-password-value":display-name "Keystore Password":type "password":required false- :visible-if {:use-keystore true}}+ :visible-if {"use-keystore" true}}{:name "keystore-value":type "textFile":treat-before-posting "base64"- :visible-if {:use-keystore true}}]+ :visible-if {"use-keystore" true}}]true]]](testing (str " with is-hosted? " is-hosted?)(mt/with-premium-features (if is-hosted? #{:hosting} #{})@@ -196,14 +196,14 @@:description "Comma separated names of schemas that should appear in Metabase":helper-text "You can use patterns like \"auth*\" to match multiple schemas":type "text"- :visible-if {:my-schema-filters-type "inclusion"}+ :visible-if {"my-schema-filters-type" "inclusion"}:required true}{:name "my-schema-filters-patterns":placeholder "E.x. public,auth*":description "Comma separated names of schemas that should NOT appear in Metabase":helper-text "You can use patterns like \"auth*\" to match multiple schemas":type "text"- :visible-if {:my-schema-filters-type "exclusion"}+ :visible-if {"my-schema-filters-type" "exclusion"}:required true}{:name "last-prop"}](driver.u/connection-props-server->clientdiff --git a/test/metabase/embedding_rest/api/embed_test.clj b/test/metabase/embedding_rest/api/embed_test.cljindex 68fec1bdf12..78142b2f30e 100644--- a/test/metabase/embedding_rest/api/embed_test.clj+++ b/test/metabase/embedding_rest/api/embed_test.clj@@ -1199,8 +1199,8 @@(deftest remove-embedding-params(testing "parameters that are not in the `embedding-params` map at all should get removed by `enabled-params`"- (is (= {:parameters []}- (#'api.embed.common/enabled-params {:parameters {:slug "foo"}} {})))))+ (is (= []+ (#'api.embed.common/enabled-params [{:id "_FOO_", :type :category, :slug "foo"}] {})))))(deftest make-sure-that-multiline-series-word-as-expected---4768-(testing "make sure that multiline series word as expected (#4768)"@@ -1807,18 +1807,20 @@(deftest apply-slug->value-test(testing "For operator filter types treat a lone value as a one-value sequence (#20438)"- (is (= (#'api.embed.common/apply-slug->value [{:type :string/=+ (is (= (#'api.embed.common/apply-slug->value [{:id "NAME"+ :type :string/=:target [:dimension [:template-tag "NAME"]]:name "Name":slug "NAME":default nil}]- {:NAME ["Aaron Hand"]})- (#'api.embed.common/apply-slug->value [{:type :string/=+ {"NAME" ["Aaron Hand"]})+ (#'api.embed.common/apply-slug->value [{:id "NAME"+ :type :string/=:target [:dimension [:template-tag "NAME"]]:name "Name":slug "NAME":default nil}]- {:NAME "Aaron Hand"})))))+ {"NAME" "Aaron Hand"})))))(deftest handle-single-params-for-operator-filters-test(testing "Query endpoints should work with a single URL parameter for an operator filter (#20438)"diff --git a/test/metabase/explorations/api_test.clj b/test/metabase/explorations/api_test.cljindex 379d4bb53c7..bfd06030fd9 100644--- a/test/metabase/explorations/api_test.clj+++ b/test/metabase/explorations/api_test.clj@@ -2519,10 +2519,10 @@:collection-id (:id coll):card-id (:id metric):database-id (mt/id)- :dimension-id "price"+ :dimension-id (duid "price"):metrics [{:card_id (:id metric)- :dimension_mappings (venues-dimension-mappings)}]- :dimensions [{:dimension_id "price" :display_name "Price"}]})+ :dimension_mappings (stored-venues-dimension-mappings)}]+ :dimensions [{:dimension-id (duid "price") :display-name "Price"}]})filter-spec {:operator "between":field_ref ["field" (mt/id :venues :price) nil]:values [1 3]diff --git a/test/metabase/explorations/interestingness_test.clj b/test/metabase/explorations/interestingness_test.cljindex fa0a29b8981..facd790c496 100644--- a/test/metabase/explorations/interestingness_test.clj+++ b/test/metabase/explorations/interestingness_test.clj@@ -224,7 +224,7 @@(let [cfg (explorations.interestingness/chart-config(query "bar")[(lib-col {:name "price_bin" :base-type :type/Integer :display-name "Price"- :lib/source :source/breakouts})+ :lib/breakout? true})(lib-col {:name "count" :base-type :type/Integer :display-name "Count":lib/source :source/aggregations})][[10 5] [20 8] [30 3]])@@ -240,9 +240,9 @@(let [cfg (explorations.interestingness/chart-config(query "line")[(lib-col {:name "bin" :base-type :type/Integer :display-name "Bin"- :lib/source :source/breakouts})+ :lib/breakout? true})(lib-col {:name "month" :base-type :type/DateTime :display-name "Month"- :lib/source :source/breakouts})+ :lib/breakout? true})(lib-col {:name "rev" :base-type :type/Integer :display-name "Revenue":lib/source :source/aggregations})][[1 "2026-01-01" 100]diff --git a/test/metabase/explorations/runner_test.clj b/test/metabase/explorations/runner_test.cljindex 2d7c9c20bfd..5a3e34cc3b9 100644--- a/test/metabase/explorations/runner_test.clj+++ b/test/metabase/explorations/runner_test.clj@@ -374,9 +374,9 @@(with-redefs [qp.context/build-row-context (constantly {::stub true})qp.variants/dataset-query (fn [_variant _ctx](reset! seen api/*current-user-id*)- (lib/->legacy-MBQL (let [mp (mt/metadata-provider)]- (-> (lib/query mp (lib.metadata/table mp (mt/id :venues)))- (lib/aggregate (lib/count))))))+ (let [mp (mt/metadata-provider)]+ (-> (lib/query mp (lib.metadata/table mp (mt/id :venues)))+ (lib/aggregate (lib/count)))))qp.variants/query-name (constantly "stub name")](runner/run-query! (:id row)))(is (= (:id u) @seen)diff --git a/test/metabase/formatter/impl_test.clj b/test/metabase/formatter/impl_test.cljindex e4b916a0ee1..89bd8a8d9f1 100644--- a/test/metabase/formatter/impl_test.clj+++ b/test/metabase/formatter/impl_test.clj@@ -288,7 +288,7 @@(deftest pivot-make-formatters-inlines-currency-test(testing "Pivot value formatters render the currency symbol inline even when currency-in-header is the default (UXW-4499).In-app pivot tables always show the currency symbol in the cell because measures have no column header to carry it."- (let [col {:name "sum" :base_type :type/Float :effective_type :type/Float :field_ref [:aggregation 0]}]+ (let [col {:name "sum" :display_name "Sum" :base_type :type/Float :effective_type :type/Float :field_ref [:aggregation 0]}](doseq [[label viz] [["currency-in-header unset (default)"{::mb.viz/column-settings{{::mb.viz/column-name "sum"} {::mb.viz/number-style "currency"diff --git a/test/metabase/indexed_entities/models/model_index_test.clj b/test/metabase/indexed_entities/models/model_index_test.cljindex f8f35ccc574..c49b59c4083 100644--- a/test/metabase/indexed_entities/models/model_index_test.clj+++ b/test/metabase/indexed_entities/models/model_index_test.clj@@ -63,7 +63,7 @@:error nil}(mt/user-http-request :rasta :get 200 (str "/model-index/" (:id model-index))))))(testing "We can invoke the task ourself manually"- (model-index/add-values! model-index)+ (model-index/add-values! (t2/select-one :model/ModelIndex :id (:id model-index)))(is (= 9 (count (t2/select :model/ModelIndexValue :model_index_id (:id model-index)))))(is (= (into #{} cat (mt/rows (qp/process-query(mt/mbql-query products {:fields [$title]@@ -79,7 +79,7 @@{:filter [:and[:> $id 10][:< $id 20]]})})- (model-index/add-values! model-index)+ (model-index/add-values! (t2/select-one :model/ModelIndex :id (:id model-index)))(is (= 9 (count (t2/select :model/ModelIndexValue :model_index_id (:id model-index)))))(is (= (into #{} cat (mt/rows (qp/process-query(mt/mbql-query products {:fields [$title]@@ -210,7 +210,8 @@(testing (str "scenario: " scenario)(let [[pk-ref value-ref] (or field-refs(->> model :result_metadata (map :field_ref)))- [error values] (#'model-index/fetch-values {:model_id (u/the-id model)+ [error values] (#'model-index/fetch-values {:id 1+ :model_id (u/the-id model):pk_ref pk-ref:value_ref value-ref})](is (nil? error))@@ -240,7 +241,7 @@:pk_ref (by-name pk-name):value_ref (by-name value-name)})];; post most likely creates this, but duplicate to be sure- (model-index/add-values! model-index)+ (model-index/add-values! (t2/select-one :model/ModelIndex :id (:id model-index)))(is (= "indexed"(t2/select-one-fn :state :model/ModelIndex :id (u/the-id model-index))))(is (= quantitydiff --git a/test/metabase/indexes/reconcile_test.clj b/test/metabase/indexes/reconcile_test.cljindex 169254c221f..222e183c146 100644--- a/test/metabase/indexes/reconcile_test.clj+++ b/test/metabase/indexes/reconcile_test.clj@@ -2,7 +2,8 @@(:require[clojure.test :refer :all][metabase.driver]- [metabase.indexes.reconcile :as reconcile]))+ [metabase.indexes.reconcile :as reconcile]+ [metabase.test :as mt]))(def ^:private managed-btree{:id 1 :transform_id 7 :index_name "by_cat"@@ -130,10 +131,10 @@(testing "delegates to the driver method"(with-redefs [metabase.driver/fetch-table-indexes (fn [_driver _db _schema _table] [wh-btree])](is (= [wh-btree]- (reconcile/fetch-warehouse-indexes {:engine :postgres} "public" "t")))))+ (reconcile/fetch-warehouse-indexes (mt/db) "public" "t")))))(testing "preserves a successful empty warehouse response"(with-redefs [metabase.driver/fetch-table-indexes (fn [& _] [])]- (is (= [] (reconcile/fetch-warehouse-indexes {:engine :postgres} "public" "t")))))+ (is (= [] (reconcile/fetch-warehouse-indexes (mt/db) "public" "t")))))(testing "swallows driver/connection errors and returns nil"(with-redefs [metabase.driver/fetch-table-indexes (fn [& _] (throw (ex-info "boom" {})))]- (is (nil? (reconcile/fetch-warehouse-indexes {:engine :postgres} "public" "t"))))))+ (is (nil? (reconcile/fetch-warehouse-indexes (mt/db) "public" "t"))))))diff --git a/test/metabase/interestingness/chart/util_test.clj b/test/metabase/interestingness/chart/util_test.cljindex 83d60646e78..53cc530b749 100644--- a/test/metabase/interestingness/chart/util_test.clj+++ b/test/metabase/interestingness/chart/util_test.clj@@ -224,6 +224,13 @@:std-dev 2.0:range 9.0})+(def ^:private sample-series-config+ {:x {:name "X" :type "string"}+ :y {:name "Y" :type "number"}+ :x_values ["a"]+ :y_values [1]+ :display_name "Test"})+(def ^:private sample-categorical-series{"s1" {:summary sample-summary:data-points 5@@ -253,12 +260,12 @@:series-count 1:series sample-categorical-series:correlations (symbol "nil #_\"key is not present.\"")}- (stats.u/make-chart-result :categorical {"s1" {}} sample-categorical-series nil)))))+ (stats.u/make-chart-result :categorical {"s1" sample-series-config} sample-categorical-series nil)))))(deftest ^:parallel make-chart-result-with-correlations-test(testing "includes correlations when provided"(let [corrs [{:series-a "a" :series-b "b" :coefficient 0.9- :strength :strong :direction :positive}]+ :strength :strong :direction :positive :aligned-sample-size 10}]series {"a" {:summary sample-summary:data-points 2:category-count 2@@ -269,10 +276,12 @@:category-count 2:top-categories []:outliers []}}- result (stats.u/make-chart-result :categorical {"a" {} "b" {}} series corrs)]+ result (stats.u/make-chart-result :categorical+ {"a" sample-series-config "b" sample-series-config}+ series corrs)](is (= corrs (:correlations result))))))(deftest ^:parallel make-chart-result-nil-correlations-test(testing "omits correlations when nil"(is (=? {:correlations (symbol "nil #_\"key is not present.\"")}- (stats.u/make-chart-result :histogram {"s1" {}} sample-histogram-series nil)))))+ (stats.u/make-chart-result :histogram {"s1" sample-series-config} sample-histogram-series nil)))))diff --git a/test/metabase/legacy_mbql/normalize_test.cljc b/test/metabase/legacy_mbql/normalize_test.cljcindex 77ede2f7c7f..29493ee8a07 100644--- a/test/metabase/legacy_mbql/normalize_test.cljc+++ b/test/metabase/legacy_mbql/normalize_test.cljc@@ -1632,9 +1632,6 @@(deftest ^:parallel normalize-datetime-test(is (= [:datetime ""](mbql.normalize/normalize [:datetime ""])))- (testing "if we add other options, they are preserved (and don't break anything)"- (is (= [:datetime "" {:x "x"}]- (mbql.normalize/normalize [:datetime "" {"x" "x"}]))))(is (= [:datetime "" {:mode :iso}](mbql.normalize/normalize [:datetime "" {:mode :iso}])(mbql.normalize/normalize [:datetime "" {:mode "iso"}])diff --git a/test/metabase/legacy_mbql/schema_test.cljc b/test/metabase/legacy_mbql/schema_test.cljcindex bf211ad7831..f011f700914 100644--- a/test/metabase/legacy_mbql/schema_test.cljc+++ b/test/metabase/legacy_mbql/schema_test.cljc@@ -145,8 +145,8 @@"[[WHERE {{date_range}}]]""ORDER BY \"TIMESTAMP\" ASC"" LIMIT 1"])- :template-tags {"date_range" template-tag}- :parameters [parameter]}}]+ :template-tags {"date_range" template-tag}}+ :parameters [parameter]}](is (nil? (me/humanize (mr/explain ::mbql.s/Query query))))))))(deftest ^:parallel value-testdiff --git a/test/metabase/legacy_mbql/util_test.cljc b/test/metabase/legacy_mbql/util_test.cljcindex 5d872e9a285..eadf25f2e38 100644--- a/test/metabase/legacy_mbql/util_test.cljc+++ b/test/metabase/legacy_mbql/util_test.cljc@@ -115,42 +115,42 @@(t/is (thrown-with-msg?#?(:clj clojure.lang.ExceptionInfo :cljs :default)#"No expression named"- (mbql.u/expression-with-name {} "wow")))))+ (mbql.u/expression-with-name {:source-table 1} "wow")))))(t/deftest ^:parallel update-field-options-test;; the options exercised here (`:display-name`, `:name`, `:binning`) are real declared ref options. Ref options are;; a closed schema in MBQL 4 as well as MBQL 5, so a stand-in key like `:wow` would be rejected as this function's;; input before it did any of the updating this test is about.(t/is (= [:field 1 {:display-name "Wow"}]- (mbql.u/update-field-options [:field 1 nil] assoc :display-name "Wow")+ (mbql.u/update-field-options [:field 1 nil] #(assoc % :display-name "Wow"))(mu/disable-enforcement- (mbql.u/update-field-options [:field 1 {}] assoc :display-name "Wow"))- (mbql.u/update-field-options [:field 1 {:display-name "Meh"}] assoc :display-name "Wow")))+ (mbql.u/update-field-options [:field 1 {}] #(assoc % :display-name "Wow")))+ (mbql.u/update-field-options [:field 1 {:display-name "Meh"}] #(assoc % :display-name "Wow"))))(t/is (= [:field 1 {:name "a", :display-name "b"}]- (mbql.u/update-field-options [:field 1 {:name "a"}] assoc :display-name "b")))+ (mbql.u/update-field-options [:field 1 {:name "a"}] #(assoc % :display-name "b"))))(t/testing "Should remove empty options"(t/is (= [:field 1 nil]- (mbql.u/update-field-options [:field 1 {:name "a"}] dissoc :name))))+ (mbql.u/update-field-options [:field 1 {:name "a"}] #(dissoc % :name)))))(t/testing "Should normalize the clause"(t/is (= [:field 1 nil](mbql.u/update-field-options [:field 1 {:binning {:strategy :default}}]- assoc-in [:binning :strategy] nil))))+ #(assoc-in % [:binning :strategy] nil)))))(t/testing "Should work with `:expression` and `:aggregation` references as well"(t/is (= [:expression "wow" {:name "a"}]- (mbql.u/update-field-options [:expression "wow"] assoc :name "a")))+ (mbql.u/update-field-options [:expression "wow"] #(assoc % :name "a"))))(t/is (= [:expression "wow" {:name "a", :display-name "b"}]- (mbql.u/update-field-options [:expression "wow" {:display-name "b"}] assoc :name "a")))+ (mbql.u/update-field-options [:expression "wow" {:display-name "b"}] #(assoc % :name "a"))))(t/is (= [:aggregation 0 {:name "a"}]- (mbql.u/update-field-options [:aggregation 0] assoc :name "a")))+ (mbql.u/update-field-options [:aggregation 0] #(assoc % :name "a"))))(t/is (= [:aggregation 0 {:name "a", :display-name "b"}]- (mbql.u/update-field-options [:aggregation 0 {:display-name "b"}] assoc :name "a")))+ (mbql.u/update-field-options [:aggregation 0 {:display-name "b"}] #(assoc % :name "a"))));; in the future when we make the 3-arg version the normalized/"official" version we will probably want to stop;; doing this.(t/testing "Remove empty options entirely from `:expression` and `:aggregation` (for now)"(t/is (= [:expression "wow"]- (mbql.u/update-field-options [:expression "wow" {:display-name "b"}] dissoc :display-name)))+ (mbql.u/update-field-options [:expression "wow" {:display-name "b"}] #(dissoc % :display-name))))(t/is (= [:aggregation 0]- (mbql.u/update-field-options [:aggregation 0 {:display-name "b"}] dissoc :display-name))))))+ (mbql.u/update-field-options [:aggregation 0 {:display-name "b"}] #(dissoc % :display-name)))))))(t/deftest ^:parallel with-temporal-unit-test(t/is (= [:field 1 {:temporal-unit :day}]diff --git a/test/metabase/lib/aggregation_test.cljc b/test/metabase/lib/aggregation_test.cljcindex a17bc826e7c..22faef5bc59 100644--- a/test/metabase/lib/aggregation_test.cljc+++ b/test/metabase/lib/aggregation_test.cljc@@ -228,9 +228,13 @@[:+ {:lib/uuid (str (random-uuid))} field 1][:- {:lib/uuid (str (random-uuid))} field 1][:* {:lib/uuid (str (random-uuid))} field 1]])- :let [clause [tag- {:lib/uuid (str (random-uuid))}- arg]]]+ :let [clause (cond-> [tag+ {:lib/uuid (str (random-uuid))}+ arg]+ (= tag :percentile) (conj 0.5)+ (= tag :sum-where) (conj [:> {:lib/uuid (str (random-uuid))}+ [:field {:lib/uuid (str (random-uuid))} (meta/id :venues :price)]+ 1]))]](testing (str \newline (pr-str clause))(is (= :type/Number(lib.schema.expression/type-of clause)))diff --git a/test/metabase/lib/column_group_test.cljc b/test/metabase/lib/column_group_test.cljcindex c0cb66802ac..d088ae4c9e1 100644--- a/test/metabase/lib/column_group_test.cljc+++ b/test/metabase/lib/column_group_test.cljc@@ -16,7 +16,7 @@(let [query (lib.tu/venues-query)columns (lib/orderable-columns query)groups (lib/group-columns columns)]- (is (not (mr/explain [:sequential @#'lib.column-group/ColumnGroup] groups)))+ (is (not (mr/explain [:sequential ::lib.column-group/column-group] groups)))(is (=? [{::lib.column-group/group-type :group-type/main:lib/type :metadata/column-group::lib.column-group/columns [{:name "ID", :display-name "ID"}diff --git a/test/metabase/lib/drill_thru/column_filter_test.cljc b/test/metabase/lib/drill_thru/column_filter_test.cljcindex c81bddb0662..44852412459 100644--- a/test/metabase/lib/drill_thru/column_filter_test.cljc+++ b/test/metabase/lib/drill_thru/column_filter_test.cljc@@ -232,15 +232,12 @@:type #:type{:DateTime {:earliest "2023-12-08T23:49:58.310952Z", :latest "2023-12-08T23:49:58.310952Z"}}}:base_type :type/DateTime}]:database-id (meta/id)- :query-type :native:dataset-query {:database (meta/id):native {:query "select 1 as \"ID\", current_timestamp::datetime as \"ALIAS_CREATED_AT\"", :template-tags {}}:type :native}:id 5- :parameter-mappings []:display :table- :visualization-settings {:table.pivot_column "ID", :table.cell_column "ALIAS_CREATED_AT"}- :parameters []}+ :visualization-settings {:table.pivot_column "ID", :table.cell_column "ALIAS_CREATED_AT"}}lib.tu/as-modellib.tu/metadata-provider-with-mock-card)query (lib/query metadata-provider (lib.metadata/card metadata-provider 5))diff --git a/test/metabase/lib/drill_thru/test_util.cljc b/test/metabase/lib/drill_thru/test_util.cljcindex 2289ea6c239..c53a65437ae 100644--- a/test/metabase/lib/drill_thru/test_util.cljc+++ b/test/metabase/lib/drill_thru/test_util.cljc@@ -4,6 +4,7 @@#?@(:cljs ([metabase.test-runner.assert-exprs.approximately-equal]))[clojure.test :refer [is testing]][clojure.walk :as walk]+ [malli.util :as mut][medley.core :as m][metabase.lib.core :as lib][metabase.lib.dispatch :as lib.dispatch]@@ -11,12 +12,15 @@[metabase.lib.metadata :as lib.metadata][metabase.lib.schema :as lib.schema][metabase.lib.schema.drill-thru :as lib.schema.drill-thru]+ [metabase.lib.schema.expression :as lib.schema.expression]+ [metabase.lib.schema.literal :as lib.schema.literal][metabase.lib.schema.metadata :as lib.schema.metadata][metabase.lib.test-metadata :as meta][metabase.lib.test-util :as lib.tu][metabase.lib.util :as lib.util][metabase.util :as u]- [metabase.util.malli :as mu]))+ [metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr]))#?(:cljs (comment metabase.test-runner.assert-exprs.approximately-equal/keep-me))@@ -69,18 +73,48 @@[:or schema [:-> schema schema]])(def ^:private Row- [:map-of :string :any])+ [:map-of :string ::lib.schema.literal/literal])-(def ^:private TestCase- [:map- [:click-type [:enum :cell :header]]+(mr/def ::expectation+ "A `=?` expectation: partial maps, predicates, and regexes matched against a real drill or query."+ [:schema {::mr/deliberately-open true, :description "a =? expectation pattern"} :any])++(def ^:private DrillArgs+ [:maybe [:sequential [:or :keyword ::lib.schema.literal/literal ::lib.schema.metadata/column]]])++(def ^:private QueryKinds+ [:sequential [:enum :mbql :native]])++(def ^:private test-case-entries+ [[:click-type [:enum :cell :header]][:query-type [:enum :aggregated :unaggregated]][:column-name :string];; defaults to "ORDERS"[:query-table {:optional true} [:maybe [:enum "ORDERS" "PRODUCTS"]]][:custom-query {:optional true} [:maybe (schema-or-update-fn ::lib.schema/query)]][:custom-native {:optional true} [:maybe (schema-or-update-fn ::lib.schema/query)]]- [:custom-row {:optional true} [:maybe (schema-or-update-fn Row)]]])+ [:custom-row {:optional true} [:maybe (schema-or-update-fn Row)]]+ [:drill-type {:optional true} ::lib.schema.drill-thru/drill-thru.type]+ [:expected {:optional true} [:maybe ::expectation]]+ [:expected-query {:optional true} [:maybe ::expectation]]+ [:expected-native {:optional true} [:maybe ::expectation]]+ [:query-kinds {:optional true} [:maybe QueryKinds]]+ [:native-drills {:optional true} ifn?]+ [:drill-args {:optional true} DrillArgs]+ [:drill-query-native {:optional true} [:maybe ::lib.schema/query]]])++(defn- test-case-schema+ "A closed test case map schema in which the optional entries named by `required-keys` are required."+ [& required-keys]+ (into [:map {:closed true}]+ (map (fn [[k :as entry]]+ (if (some #{k} required-keys)+ [k (peek entry)]+ entry)))+ test-case-entries))++(def ^:private TestCase+ (test-case-schema))(def ^:private ^:dynamic *native-card-id* nil)@@ -177,7 +211,7 @@(append-filter-stage query column-name #(lib/> % -1)))([query :- ::lib.schema/querycolumn-name :- :string- column-filter-fn :- [:-> ::lib.schema.metadata/column :any]]+ column-filter-fn :- [:-> ::lib.schema.metadata/column ::lib.schema.expression/boolean]](let [query' (lib/append-stage query)column-to-filter (column-by-name query' column-name)](assert (some? column-to-filter) (str "Failed to find " column-name " in " query))@@ -213,7 +247,7 @@(append-filter-stage-to-test-expectation expected-query my-column-name)are matching pairs."- ([expected-query :- :map+ ([expected-query :- ::expectationfield-matcher-or-filter-expr :- FieldMatcherOrFilterExpr](assert (vector? (:stages expected-query))"expected-query should have a :stages key mapped to a vector")@@ -225,21 +259,21 @@Useful for updating the `:expected-query` for [[test-drill-application]] when the `:custom-query` was modifiedby [[append-filter-stage]]."- ([expected-query :- :map+ ([expected-query :- ::expectationfield-matcher-or-filter-expr :- FieldMatcherOrFilterExpr](prepend-filter-to-test-expectation-stage expected-query -1 field-matcher-or-filter-expr))- ([expected-query :- :map+ ([expected-query :- ::expectationstage-number :- :intfield-matcher-or-filter-expr :- FieldMatcherOrFilterExpr](assert (vector? (:stages expected-query))"expected-query should have a :stages key mapped to a vector")(let [filter-expr (field-matcher-or-filter-expr->filter-expr field-matcher-or-filter-expr)](update-in expected-query- [:stages (lib.util/canonical-stage-index expected-query stage-number) :filters]+ [:stages (cond-> stage-number (neg? stage-number) (+ (count (:stages expected-query)))) :filters]#(into [filter-expr] %)))))(mu/defn prepend-stage-to-test-expectation- [expected-query :- :map]+ [expected-query :- ::expectation](assert (vector? (:stages expected-query))"expected-query should have a :stages key mapped to a vector")(update expected-query :stages #(into [{}] %)))@@ -279,7 +313,10 @@:row row}))(mu/defn test-case-context :- ::lib.schema.drill-thru/context- [{:keys [mbql row]} :- [:map] ;; TODO: Better type? Does one exist?+ [{:keys [mbql row]} :- [:map {:closed true}+ [:mbql ::lib.schema/query]+ [:native {:optional true} [:maybe ::lib.schema/query]]+ [:row Row]]query-kind :- [:enum :mbql :native]{:keys [column-name click-type query-type], :as _test-case} :- TestCase](let [returned (lib/returned-columns mbql -1 (lib.util/query-stage mbql -1))@@ -324,14 +361,7 @@:dimensions dimensions}))))(def ^:private AvailableDrillsTestCase- [:merge- TestCase- [:map- [:expected [:or [:-> [:sequential [:map- [:type ::lib.schema.drill-thru/drill-thru.type]]]- :boolean]- [:sequential [:map- [:type ::lib.schema.drill-thru/drill-thru.type]]]]]]])+ (test-case-schema :expected))(mu/defn test-available-drill-thrus[{:keys [column-name click-type query-type query-table query-kinds expected native-drills]@@ -354,17 +384,10 @@(lib/available-drill-thrus query -1 context))))))))(def ^:private TestCaseWithDrillType- [:merge- TestCase- [:map- [:drill-type ::lib.schema.drill-thru/drill-thru.type]]])+ (test-case-schema :drill-type))(def ^:private ReturnsDrillTestCase- [:merge- TestCaseWithDrillType- [:map- [:expected [:map- [:type ::lib.schema.drill-thru/drill-thru.type]]]]])+ (test-case-schema :drill-type :expected))(defn- drop-uuids [form](walk/postwalk #(cond-> % (map? %) (dissoc :lib/uuid))@@ -442,11 +465,7 @@(is (not (contains? drills drill-type)))))))))(def ^:private DrillApplicationTestCase- [:merge- ReturnsDrillTestCase- [:map- [:expected-query :map]- [:drill-args {:optional true} [:maybe [:sequential :any]]]]])+ (test-case-schema :drill-type :expected :expected-query))(mu/defn test-drill-application"Test that a certain drill gets returned, AND when applied to a query returns the expected query."@@ -470,6 +489,10 @@(clean-expected-query expected-query))query'))))))))+(def ^:private PartialTestCase+ "A partial `TestCase` map for a `variant-case` that gets merged onto a `base-case`."+ (mut/optional-keys TestCase))+(mu/defn test-drill-variants-with-merged-args"Run `test-fn` first with `base-case` then with each of the specified `variants`.@@ -492,10 +515,10 @@If any `variant-case` is a fn, it should be of type map -> map and will be passed the `base-case` and the returnedmap will be merged with `base-case` instead."- [test-fn :- [:-> :map :any]+ [test-fn :- [:-> TestCase [:maybe ::lib.schema.drill-thru/drill-thru]]base-desc :- :string- base-case :- :map- & variants]+ base-case :- TestCase+ & variants :- [:* [:or :string PartialTestCase [:-> PartialTestCase PartialTestCase]]]](assert (even? (count variants)) "variants must come in variant-desc and variant-case pairs")(when-not (= "SKIP" base-desc)diff --git a/test/metabase/lib/expression_test.cljc b/test/metabase/lib/expression_test.cljcindex fcdacbdbfba..8d383b274d2 100644--- a/test/metabase/lib/expression_test.cljc+++ b/test/metabase/lib/expression_test.cljc@@ -119,7 +119,7 @@(deftest ^:parallel date-interval-names-test(let [clause [:datetime-add- {}+ {:lib/uuid (str (random-uuid))}(lib.tu/field-clause :checkins :date {:base-type :type/Date})-1:day]]@@ -130,7 +130,7 @@(deftest ^:parallel datetime-subtract-names-test(let [clause [:datetime-subtract- {}+ {:lib/uuid (str (random-uuid))}(lib.tu/field-clause :checkins :date {:base-type :type/Date})1:day]]@@ -154,7 +154,7 @@(lib/column-name query -1 expr)))))(deftest ^:parallel coalesce-names-test- (let [clause [:coalesce {} (lib.tu/field-clause :venues :name) "<Venue>"]]+ (let [clause [:coalesce {:lib/uuid (str (random-uuid))} (lib.tu/field-clause :venues :name) "<Venue>"]](is (= "NAME"(lib/column-name (lib.tu/venues-query) -1 clause)))(is (= "Name"@@ -541,13 +541,13 @@(is (= (count exprs) (count expressions)))(is (some? c-pos))(testing "no circularity problem"- (are [mode expr]- (nil? (lib.expression/diagnose-expression query 0 mode expr c-pos))- :expression (get exprs "non-circular-c")+ (are [mode expr position]+ (nil? (lib.expression/diagnose-expression query 0 mode expr position))+ :expression (get exprs "non-circular-c") c-pos:aggregation (-> (get exprs "circular-c")(update 2 lib/sum)- (assoc 4 (lib/count)))- :filter (assoc (get exprs "circular-c") 0 :=)))+ (assoc 4 (lib/count))) nil+ :filter (assoc (get exprs "circular-c") 0 :=) c-pos))(testing "circular definition"(is (=? {:message "Cycle detected: c → x → b → c"}(lib.expression/diagnose-expression query 0 :expressiondiff --git a/test/metabase/lib/fe_util_test.cljc b/test/metabase/lib/fe_util_test.cljcindex ac68bcb8103..5df7a741ad0 100644--- a/test/metabase/lib/fe_util_test.cljc+++ b/test/metabase/lib/fe_util_test.cljc@@ -166,7 +166,7 @@segments-db {:segments [{:id segment-id:name segment-name:table-id (meta/id :venues)- :definition {}+ :definition nil:description segment-description}]}metadata-provider (lib.tu/mock-metadata-provider meta/metadata-provider segments-db)query (lib/query metadata-provider (meta/table-metadata :venues))@@ -192,7 +192,7 @@metrics-db {:segments [{:id metric-id:name metric-name:table-id (meta/id :venues)- :definition {}+ :definition nil:description metric-description}]}metadata-provider (lib.tu/mock-metadata-provider meta/metadata-provider metrics-db)query (lib/query metadata-provider (meta/table-metadata :venues))diff --git a/test/metabase/lib/field/resolution_test.cljc b/test/metabase/lib/field/resolution_test.cljcindex 639e3d4a47f..903e09a78fc 100644--- a/test/metabase/lib/field/resolution_test.cljc+++ b/test/metabase/lib/field/resolution_test.cljc@@ -1915,15 +1915,14 @@:lib/original-join-alias "Products":lib/original-name "ID_2":lib/type :metadata/column}- (lib/metadata query -1- (#'lib.field.resolution/additional-metadata-from-source-card- query- -1- (lib/normalize- ::lib.schema.metadata/column- {:lib/type :metadata/column- :lib/source :source/previous-stage- :id (meta/id :products :id)- :name "Products__ID"- :lib/source-column-alias "Products__ID"- :base-type :type/Integer}))))))))+ (#'lib.field.resolution/additional-metadata-from-source-card+ query+ -1+ (lib/normalize+ ::lib.schema.metadata/column+ {:lib/type :metadata/column+ :lib/source :source/previous-stage+ :id (meta/id :products :id)+ :name "Products__ID"+ :lib/source-column-alias "Products__ID"+ :base-type :type/Integer})))))))diff --git a/test/metabase/lib/measure_test.cljc b/test/metabase/lib/measure_test.cljcindex 4e13dedd8d1..fe05d4eb7fb 100644--- a/test/metabase/lib/measure_test.cljc+++ b/test/metabase/lib/measure_test.cljc@@ -216,7 +216,7 @@(let [mp (lib.tu/mock-metadata-provider meta/metadata-provider {})query (lib/query mp (meta/table-metadata :venues))](is (= :type/*- (lib/type-of query [:measure {} 999]))))))+ (lib/type-of query [:measure {:lib/uuid (str (random-uuid))} 999]))))))(def ^:private measure-id 100)diff --git a/test/metabase/lib/metadata/calculation_test.cljc b/test/metabase/lib/metadata/calculation_test.cljcindex d2145d3286f..62c043d89bc 100644--- a/test/metabase/lib/metadata/calculation_test.cljc+++ b/test/metabase/lib/metadata/calculation_test.cljc@@ -1254,21 +1254,22 @@#?(:clj(deftest ^:parallel display-name-vm-error-test- (testing "a VM Error thrown while computing a display name propagates unwrapped"- (let [e (Error. "boom")]- (is (identical? e- (try- (lib.metadata.calculation/display-name (lib.tu/venues-query)- {:lib/type ::throws, :throwable e})- nil- (catch Error actual actual))))))- (testing "an Exception thrown while computing a display name is wrapped with context"- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Error calculating display name"+ (mu/disable-enforcement+ (testing "a VM Error thrown while computing a display name propagates unwrapped"+ (let [e (Error. "boom")]+ (is (identical? e+ (try(lib.metadata.calculation/display-name (lib.tu/venues-query)- {:lib/type ::throws- :throwable (ex-info "boom" {})}))))- (testing "an AssertionError thrown while computing a display name is contained (wrapped), not propagated"- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Error calculating display name"- (lib.metadata.calculation/display-name (lib.tu/venues-query)- {:lib/type ::throws- :throwable (AssertionError. "boom")}))))))+ {:lib/type ::throws, :throwable e})+ nil+ (catch Error actual actual))))))+ (testing "an Exception thrown while computing a display name is wrapped with context"+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Error calculating display name"+ (lib.metadata.calculation/display-name (lib.tu/venues-query)+ {:lib/type ::throws+ :throwable (ex-info "boom" {})}))))+ (testing "an AssertionError thrown while computing a display name is contained (wrapped), not propagated"+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Error calculating display name"+ (lib.metadata.calculation/display-name (lib.tu/venues-query)+ {:lib/type ::throws+ :throwable (AssertionError. "boom")})))))))diff --git a/test/metabase/lib/metadata/result_metadata_test.cljc b/test/metabase/lib/metadata/result_metadata_test.cljcindex f3ad9dafd99..10739e58504 100644--- a/test/metabase/lib/metadata/result_metadata_test.cljc+++ b/test/metabase/lib/metadata/result_metadata_test.cljc@@ -20,7 +20,11 @@#?(:cljs (comment metabase.test-runner.assert-exprs.approximately-equal/keep-me))-(mu/defn- column-info [query :- ::lib.schema/query {initial-columns :cols}]+(mu/defn- column-info+ [query :- ::lib.schema/query+ {initial-columns :cols} :- [:maybe [:map {:closed true}+ [:cols {:optional true} ::result-metadata/initial-cols]+ [:columns {:optional true} [:sequential :keyword]]]]](result-metadata/returned-columns query initial-columns))(deftest ^:parallel col-info-field-ids-test@@ -196,7 +200,7 @@:visibility-type :normal:display-name "Grandparent: Parent":base-type :type/Text}- (first (column-info query {:cols [{:lib/transformation-added-base-type true}]})))))))+ (first (column-info query {:cols [{}]})))))))(deftest ^:parallel col-info-combine-grandparent-field-names-test(testing "nested-nested fields should include grandparent name (etc)"@@ -211,7 +215,7 @@:visibility-type :normal:display-name "Grandparent: Parent: Child":base-type :type/Text}- (first (column-info query {:cols [{:lib/transformation-added-base-type false}]})))))))+ (first (column-info query {:cols [{}]})))))))(deftest ^:parallel col-info-field-literals-test(testing "field literals should get the information from the matching `:source-metadata` if it was supplied"diff --git a/test/metabase/lib/metadata_test.cljc b/test/metabase/lib/metadata_test.cljcindex 9c7731a72af..f9135faf320 100644--- a/test/metabase/lib/metadata_test.cljc+++ b/test/metabase/lib/metadata_test.cljc@@ -20,8 +20,9 @@(deftest ^:parallel display-name-from-name-test(testing "Use the 'simple humanization' logic to calculate a display name for a Field that doesn't have one"(is (= "Venue ID"- (lib/display-name (lib.tu/venues-query) -1 {:lib/type :metadata/column- :name "venue_id"})))))+ (lib/display-name (lib.tu/venues-query) -1 {:lib/type :metadata/column+ :name "venue_id"+ :base-type :type/Integer})))))(deftest ^:parallel table-or-card-test(are [id expected] (=? expecteddiff --git a/test/metabase/lib/metric_test.cljc b/test/metabase/lib/metric_test.cljcindex 2539d14b72c..c3f2b61f372 100644--- a/test/metabase/lib/metric_test.cljc+++ b/test/metabase/lib/metric_test.cljc@@ -88,7 +88,7 @@(lib/display-name query-with-metric metric)))))))(deftest ^:parallel unknown-display-name-test- (let [metric [:metric {} 1]]+ (let [metric [:metric {:lib/uuid (str (random-uuid))} 1]](doseq [style [nil:default:long]]@@ -121,7 +121,7 @@(is (=? {:effective-type :type/*:display-name "[Unknown Metric]":long-display-name "[Unknown Metric]"}- (lib/display-info query-with-metric [:metric {} 1]))))+ (lib/display-info query-with-metric [:metric {:lib/uuid (str (random-uuid))} 1]))))(deftest ^:parallel type-of-test(are [metric] (= :type/Integer@@ -152,7 +152,7 @@(deftest ^:parallel unknown-type-of-test(is (= :type/*- (lib/type-of query-with-metric [:metric {} 1]))))+ (lib/type-of query-with-metric [:metric {:lib/uuid (str (random-uuid))} 1]))))(deftest ^:parallel display-info-unselected-metric-test(testing "Include `:selected false` in display info for Metrics not in aggregations"diff --git a/test/metabase/lib/native_test.cljc b/test/metabase/lib/native_test.cljcindex ef8f2bd40c3..bfa0ea3a221 100644--- a/test/metabase/lib/native_test.cljc+++ b/test/metabase/lib/native_test.cljc@@ -58,8 +58,7 @@(deftest ^:parallel template-tags-test(testing "snippet tags"- (let [snippet {:type :snippet- :name "foo"+ (let [snippet {:name "foo":id 1}metadata-provider (lib.tu/mock-metadata-provider{:native-query-snippets [snippet]})]@@ -134,12 +133,10 @@:type :card:card-id 321})metadata-provider (lib.tu/mock-metadata-provider- {:native-query-snippets [{:name "first snippet"- :id 123- :snippet-name "first snippet"}- {:name "another snippet"- :id 124- :snippet-name "another snippet"}]})]+ {:native-query-snippets [{:name "first snippet"+ :id 123}+ {:name "another snippet"+ :id 124}]})](is (=? [c1 v1 s1](lib.native/extract-template-tagsmetadata-provider@@ -314,7 +311,6 @@meta/metadata-provider{:native-query-snippets [{:id 1:name "snippet1"- :type :snippet:content "{{var}}":template-tags {"var" {:id "ac8a19f0-e125-418a-81dc-aa7f4f2c3e76":name "var"@@ -355,11 +351,9 @@meta/metadata-provider{:native-query-snippets [{:id 1:name "snippet1"- :type :snippet:content "SELECT"}{:id 2:name "snippet2"- :type :snippet:content "{{var}}":template-tags {"var" {:id "ac8a19f0-e125-418a-81dc-aa7f4f2c3e76":name "var"diff --git a/test/metabase/lib/options_test.cljc b/test/metabase/lib/options_test.cljcindex 9df33dbe9ba..e59093e2a4e 100644--- a/test/metabase/lib/options_test.cljc+++ b/test/metabase/lib/options_test.cljc@@ -6,21 +6,22 @@(deftest ^:parallel default-mbql-options-test(testing `lib.options/options(is (= nil- (lib.options/options [:mbql-clause 1])))- (is (= {:x 1}- (lib.options/options [:mbql-clause {:x 1} 1]))))+ (lib.options/options [:expression "x"])))+ (is (= {:lib/expression-name "x"}+ (lib.options/options [:expression {:lib/expression-name "x"} "x"]))))(testing `lib.options/with-options- (is (= [:mbql-clause {:x 2} 1]- (lib.options/with-options [:mbql-clause 1] {:x 2})- (lib.options/with-options [:mbql-clause {:x 1, :y 2} 1] {:x 2})))))+ (is (= [:expression {:name "y"} "x"]+ (lib.options/with-options [:expression {:lib/expression-name "x"} "x"] {:name "y"})))))(deftest ^:parallel default-map-options-test(testing `lib.options/options(is (= nil- (lib.options/options {:lib/type :map})))- (is (= {:x 1}- (lib.options/options {:lib/type :map, :lib/options {:x 1}}))))+ (lib.options/options {:lib/type :metadata/column, :name "x", :base-type :type/Text})))+ (is (= {:lib/expression-name "x"}+ (lib.options/options {:lib/type :metadata/column, :name "x", :base-type :type/Text+ :lib/options {:lib/expression-name "x"}}))))(testing `lib.options/with-options- (is (= {:lib/type :map, :lib/options {:x 2}}- (lib.options/with-options {:lib/type :map, :lib/options {:x 1}} {:x 2})- (lib.options/with-options {:lib/type :map, :lib/options {:x 1, :y 2}} {:x 2})))))+ (is (= {:lib/type :metadata/column, :name "x", :base-type :type/Text, :lib/options {:lib/expression-name "y"}}+ (lib.options/with-options {:lib/type :metadata/column, :name "x", :base-type :type/Text+ :lib/options {:lib/expression-name "x"}}+ {:lib/expression-name "y"})))))diff --git a/test/metabase/lib/parameters_test.cljc b/test/metabase/lib/parameters_test.cljcindex f0c492cf032..c243ac3756e 100644--- a/test/metabase/lib/parameters_test.cljc+++ b/test/metabase/lib/parameters_test.cljc@@ -45,10 +45,4 @@(is (= target(lib/update-parameter-target-field-reftarget- (fn [_] (throw (ex-info "should not be called" {}))))))))- (testing "passes extra args to f"- (is (= [:dimension other-ref]- (lib/update-parameter-target-field-ref- [:dimension field-ref]- (fn [_mbql5-ref extra-col] (lib.ref/ref extra-col))- other-col))))))+ (fn [_] (throw (ex-info "should not be called" {}))))))))))diff --git a/test/metabase/lib/remove_replace_test.cljc b/test/metabase/lib/remove_replace_test.cljcindex e0a009ef784..3663a8b1e72 100644--- a/test/metabase/lib/remove_replace_test.cljc+++ b/test/metabase/lib/remove_replace_test.cljc@@ -586,12 +586,14 @@meta/metadata-provider{:segments [{:id 100:name "Price is 4"- :definition {:filter+ :definition {:source-table (meta/id :venues)+ :filter[:= [:field (meta/id :venues :price) nil] 4]}:table-id (meta/id :venues)}{:id 200:name "Price is 5"- :definition {:filter+ :definition {:source-table (meta/id :venues)+ :filter[:= [:field (meta/id :venues :price) nil] 5]}:table-id (meta/id :venues)}]})query (-> (lib/query metadata-provider (meta/table-metadata :venues))diff --git a/test/metabase/lib/schema/expression/conditional_test.cljc b/test/metabase/lib/schema/expression/conditional_test.cljcindex ae578c0da77..dad4a7503d0 100644--- a/test/metabase/lib/schema/expression/conditional_test.cljc+++ b/test/metabase/lib/schema/expression/conditional_test.cljc@@ -2,8 +2,10 @@(:require[clojure.test :refer [are deftest is testing]][metabase.lib.schema :as lib.schema]+ [metabase.lib.schema.common :as lib.schema.common][metabase.lib.schema.expression :as expression][metabase.lib.schema.expression.conditional :as expression.conditional]+ [metabase.lib.schema.literal :as lib.schema.literal][metabase.lib.test-metadata :as meta][metabase.test-runner.assert-exprs.malli-equals][metabase.util :as u]@@ -17,7 +19,7 @@(#'expression.conditional/best-return-type :type/Integer ::expression/type.unknown))))(mu/defn- case-expr :- :mbql.clause/case- [& args]+ [& args :- [:* ::expression/expression]][:case{:lib/uuid (str (random-uuid))}(mapv (fn [arg]@@ -26,7 +28,8 @@args)])(mu/defn- value-expr :- :mbql.clause/value- [effective-type x]+ [effective-type :- ::lib.schema.common/base-type+ x :- ::lib.schema.literal/literal][:value {:lib/uuid (str (random-uuid)):effective-type effective-type:base-type effective-type}@@ -125,7 +128,8 @@"bar"]]}]})))(mu/defn- coalesce-expr :- :mbql.clause/coalesce- [not-nil-expr nil-expr]+ [not-nil-expr :- ::expression/expression+ nil-expr :- ::expression/expression][:coalesce{:lib/uuid (str (random-uuid))}not-nil-exprdiff --git a/test/metabase/lib/segment_test.cljc b/test/metabase/lib/segment_test.cljcindex e14f5802608..b5bcd71acfb 100644--- a/test/metabase/lib/segment_test.cljc+++ b/test/metabase/lib/segment_test.cljc@@ -65,7 +65,7 @@(is (=? {:effective-type :type/Boolean:display-name "[Unknown Segment]":long-display-name "[Unknown Segment]"}- (lib.metadata.calculation/display-info query-with-segment [:segment {} (inc segment-id)]))))+ (lib.metadata.calculation/display-info query-with-segment [:segment {:lib/uuid (str (random-uuid))} (inc segment-id)]))))(deftest ^:parallel available-segments-test(testing "Should return Segments with the same Table ID as query's `:source-table`"diff --git a/test/metabase/lib/table_test.cljc b/test/metabase/lib/table_test.cljcindex baafd1f7ecd..b614ad92b18 100644--- a/test/metabase/lib/table_test.cljc+++ b/test/metabase/lib/table_test.cljc@@ -71,14 +71,14 @@;; the database that ran the query and we need to use the original name to refer back to it in subsequent stages.(let [mp (lib.tu/mock-metadata-provider{:database (assoc meta/database :id 1)- :tables [(assoc (meta/table-metadata :venues) :id 1, :database_id 1)]+ :tables [(assoc (meta/table-metadata :venues) :id 1, :db-id 1)]:fields [(assoc (meta/field-metadata :venues :id):id 1- :table_id 1+ :table-id 1:name "Total_number_of_people_from_each_state_separated_by_state_and_then_we_do_a_count")(assoc (meta/field-metadata :venues :id):id 2- :table_id 1+ :table-id 1:name "Total_number_of_people_from_each_state_separated_by_state_and_then_we_do_a_count")]})query (lib/query mp (lib.metadata/table mp 1))](is (=? [{:lib/source-column-alias "Total_number_of_people_from_each_state_separated_by_state_and_then_we_do_a_count"diff --git a/test/metabase/lib/test_util.cljc b/test/metabase/lib/test_util.cljcindex 95910619660..50ee07366f0 100644--- a/test/metabase/lib/test_util.cljc+++ b/test/metabase/lib/test_util.cljc@@ -381,9 +381,7 @@This is mostly around for historic reasons; consider using [[metabase.lib.core/query]] instead, which is closer toreal-life usage."[metadata-providerable :- ::lib.schema.metadata/metadata-providerable- {mbql-query :dataset-query, metadata :result-metadata} :- [:map- [:dataset-query :map]- [:result-metadata [:sequential {:min 1} :map]]]]+ {mbql-query :dataset-query, metadata :result-metadata} :- ::lib.schema.metadata/card](let [mbql-query (cond-> (assoc (lib.convert/->mbql5 mbql-query):lib/metadata (lib.metadata/->metadata-provider metadata-providerable))metadata@@ -396,7 +394,11 @@:stages [{:lib/type :mbql.stage/native:native "SELECT * FROM VENUES;"}]}(query-with-stage-metadata-from-card meta/metadata-provider- {:dataset-query {:database (meta/id)+ {:lib/type :metadata/card+ :id 1+ :name "Venues"+ :database-id (meta/id)+ :dataset-query {:database (meta/id):type :native:native {:query "SELECT * FROM VENUES;"}}:result-metadata (get-in (mock-cards) [:venues :result-metadata])}))))diff --git a/test/metabase/lib/test_util/macros.clj b/test/metabase/lib/test_util/macros.cljindex 1b7fdfe86c3..7543efc67dd 100644--- a/test/metabase/lib/test_util/macros.clj+++ b/test/metabase/lib/test_util/macros.clj@@ -5,8 +5,7 @@[metabase.lib.test-metadata :as meta][metabase.lib.test-util.macros.impl :as lib.tu.macros.impl][metabase.test.data.mbql-query-impl :as mbql-query-impl]- [metabase.util :as u]- [metabase.util.malli :as mu]))+ [metabase.util :as u]))(defn- do-with-bindings [thunk](binding [mbql-query-impl/*id-fn-symb* 'metabase.lib.test-metadata/id@@ -42,9 +41,8 @@(mbql-query-impl/maybe-add-source-table <> table-name)(mbql-query-impl/wrap-inner-query <>)))))-(mu/defn- maybe-add-source-table-mbql5 :- :map- [query :- :map- table-name]+(defn- maybe-add-source-table-mbql5+ [query table-name](cond;; `table-name` is not specified: return query as is(not table-name)diff --git a/test/metabase/lib/test_util/metadata_providers/merged_mock.cljc b/test/metabase/lib/test_util/metadata_providers/merged_mock.cljcindex 689bb2ce425..466d3e9534d 100644--- a/test/metabase/lib/test_util/metadata_providers/merged_mock.cljc+++ b/test/metabase/lib/test_util/metadata_providers/merged_mock.cljc@@ -2,8 +2,8 @@(:require#?@(:cljs ([metabase.test-runner.assert-exprs.approximately-equal]))[clojure.test :refer [deftest is]]+ [malli.util :as mut][metabase.lib.metadata :as lib.metadata]- [metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.metadata :as lib.schema.metadata][metabase.lib.test-metadata :as meta][metabase.lib.test-util.metadata-providers.mock@@ -22,11 +22,11 @@(def ^:private MergeableProperties[:map{:closed true}- [:database {:optional true} [:maybe :map]]- [:tables {:optional true} [:maybe [:sequential [:map [:id ::lib.schema.id/table]]]]]- [:fields {:optional true} [:maybe [:sequential [:map [:id ::lib.schema.id/field]]]]]- [:cards {:optional true} [:maybe [:sequential [:map [:id ::lib.schema.id/card]]]]]- [:segments {:optional true} [:maybe [:sequential [:map [:id ::lib.schema.id/segment]]]]]])+ [:database {:optional true} [:maybe (mut/optional-keys ::lib.schema.metadata/database)]]+ [:tables {:optional true} [:maybe [:sequential (mut/optional-keys ::lib.schema.metadata/table)]]]+ [:fields {:optional true} [:maybe [:sequential (mut/optional-keys ::lib.schema.metadata/column.map)]]]+ [:cards {:optional true} [:maybe [:sequential (mut/optional-keys ::lib.schema.metadata/card)]]]+ [:segments {:optional true} [:maybe [:sequential (mut/optional-keys ::lib.schema.metadata/segment)]]]])(mu/defn- merged-metadata-map :- ::lib.tu.metadata-providers.mock/mock-metadata[parent-metadata-provider :- ::lib.schema.metadata/metadata-providerdiff --git a/test/metabase/lib/test_util/metadata_providers/mock.cljc b/test/metabase/lib/test_util/metadata_providers/mock.cljcindex 573d04a8894..07a13f326e2 100644--- a/test/metabase/lib/test_util/metadata_providers/mock.cljc+++ b/test/metabase/lib/test_util/metadata_providers/mock.cljc@@ -32,7 +32,7 @@[:tables {:optional true}[:maybe [:sequential (with-optional-lib-type ::lib.schema.metadata/table :metadata/table)]]][:fields {:optional true}- [:maybe [:sequential (with-optional-lib-type ::lib.schema.metadata/column :metadata/column)]]]+ [:maybe [:sequential (with-optional-lib-type ::lib.schema.metadata/column.map :metadata/column)]]][:cards {:optional true}[:maybe [:sequential (with-optional-lib-type ::lib.schema.metadata/card :metadata/card)]]][:segments {:optional true}@@ -42,9 +42,9 @@[:native-query-snippets {:optional true}[:maybe [:sequential (with-optional-lib-type ::lib.schema.metadata/native-query-snippet :metadata/native-query-snippet)]]][:transforms {:optional true}- [:maybe [:sequential (with-optional-lib-type :map :metadata/transform)]]]+ [:maybe [:sequential (with-optional-lib-type ::lib.schema.metadata/transform :metadata/transform)]]][:settings {:optional true}- [:maybe [:map-of :keyword any?]]]])+ [:maybe :metabase.lib.schema.common/database-settings]]])(defn- mock-database [metadata](some-> (:database metadata)@@ -100,6 +100,10 @@;;; like mock a Card but don't want to manually specify a `:name` every time. Just add a `:decode/mock` key to the;;; schema properties.+(mr/def ::mock-metadata-spec+ "A partial [[::mock-metadata]] map, which [[mock-coercer]] completes with mock values."+ [:map {:closed false, ::mr/deliberately-open true, :description "mock metadata spec"}])+(mu/defn- mock-coercer :- [:=> [:cat] [:=> [:cat :map] ::mock-metadata]][](mr/cached@@ -109,7 +113,7 @@(mc/coercer ::mock-metadata (mtx/transformer {:name :mock}) #_respond identity #_raise :value))))(mu/defn- ->mock-metadata :- ::mock-metadata- [m]+ [m :- [:maybe ::mock-metadata-spec]](->> m((mock-coercer))(lib.normalize/normalize ::mock-metadata)))@@ -129,12 +133,13 @@(lib.tu/mock-metadata-provider parent-metadata-provider {...})=>(lib/composed-metadata-provider (lib.tu/mock-metadata-provider {...}) parent-metadata-provider)"- ([m]+ ([m :- [:maybe ::mock-metadata-spec]](-> m->mock-metadata->MockMetadataProvider))- ([parent-metadata-provider mock-metadata]+ ([parent-metadata-provider :- ::lib.schema.metadata/metadata-providerable+ mock-metadata :- [:maybe ::mock-metadata-spec]](lib/composed-metadata-provider(mock-metadata-provider mock-metadata)parent-metadata-provider)))diff --git a/test/metabase/lib/test_util/metadata_providers/remap.cljc b/test/metabase/lib/test_util/metadata_providers/remap.cljcindex 56cc34247c1..309b8fe156b 100644--- a/test/metabase/lib/test_util/metadata_providers/remap.cljc+++ b/test/metabase/lib/test_util/metadata_providers/remap.cljc@@ -9,6 +9,21 @@[metabase.util :as u][metabase.util.malli :as mu]))+(def ^:private InternalRemap+ "Hardcoded remap values: a sequence of human-readable values keyed positionally to the original Field's values, or a+ map of original value to human-readable value."+ [:or+ [:sequential :string]+ [:map-of :metabase.lib.schema.literal/literal :string]])++(def ^:private Remap+ "Something [[remapped-column]] can remap `original` to: a remapped Field (metadata or ID) for an external remap, or+ hardcoded values for an internal remap."+ [:or+ ::lib.schema.metadata/column+ ::lib.schema.id/field+ InternalRemap])+(mu/defn- external-remapped-column :- ::lib.schema.metadata/column"Add an 'external' 'Human Readable Values' remap from values of `original` Field to values of `remapped` Field."[metadata-provider :- ::lib.schema.metadata/metadata-provider@@ -26,9 +41,7 @@(mu/defn- internal-remapped-column :- ::lib.schema.metadata/column"Add an 'internal' 'FieldValues' remap from values of `original` Field to hardcoded `remap` values."[original :- ::lib.schema.metadata/column- remap :- [:or- [:sequential :any]- :map]]+ remap :- InternalRemap](let [original-values (if (sequential? remap)(range 1 (inc (count remap)))(keys remap))@@ -51,11 +64,7 @@* Map of original value => remapped value: internal remap with values replaced with corresponding value in map"[metadata-provider :- ::lib.schema.metadata/metadata-provideroriginal :- [:or ::lib.schema.metadata/column ::lib.schema.id/field]- remap :- [:or- ::lib.schema.metadata/column- ::lib.schema.id/field- [:sequential :any]- :map]]+ remap :- Remap](let [original (if (integer? original)(lib.metadata/field metadata-provider original)original)]@@ -70,15 +79,14 @@create an internal remap."([metadata-provider :- ::lib.schema.metadata/metadata-provideroriginal :- [:or ::lib.schema.metadata/column ::lib.schema.id/field]- remap :- [:or- ::lib.schema.metadata/column- ::lib.schema.id/field- [:sequential :any]- :map]]+ remap :- Remap](let [original' (remapped-column metadata-provider original remap)](lib.tu.metadata-providers.mock/mock-metadata-providermetadata-provider{:fields [original']})))- ([metadata-provider original remap & more]+ ([metadata-provider :- ::lib.schema.metadata/metadata-provider+ original :- [:or ::lib.schema.metadata/column ::lib.schema.id/field]+ remap :- Remap+ & more :- [:* Remap]](apply remap-metadata-provider (remap-metadata-provider metadata-provider original remap) more)))diff --git a/test/metabase/lib/test_util/metadata_providers/with_cards_for_queries.cljc b/test/metabase/lib/test_util/metadata_providers/with_cards_for_queries.cljcindex 6f72bd6423d..6111067308e 100644--- a/test/metabase/lib/test_util/metadata_providers/with_cards_for_queries.cljc+++ b/test/metabase/lib/test_util/metadata_providers/with_cards_for_queries.cljc@@ -12,7 +12,7 @@"Create a metadata provider that adds a Card for each query in `queries`. Cards do not include resultmetadata. Cards have IDs starting at `1` and increasing sequentially."[parent-metadata-provider :- ::lib.schema.metadata/metadata-provider- queries :- [:sequential {:min 1} :map]]+ queries :- [:sequential {:min 1} ::lib.schema.metadata/card.query]](lib.tu.metadata-providers.mock/mock-metadata-providerparent-metadata-provider{:cards (into []@@ -23,6 +23,10 @@:database-id (or (when (pos-int? database-id)database-id)(u/the-id (lib.metadata/database parent-metadata-provider)))- :table-id (lib.util/source-table-id query)+ :table-id (if (:lib/type query)+ (lib.util/source-table-id query)+ (let [table-id (get-in query [:query :source-table])]+ (when (pos-int? table-id)+ table-id))):dataset-query query}))queries)}))diff --git a/test/metabase/lib/test_util/notebook_helpers.cljc b/test/metabase/lib/test_util/notebook_helpers.cljcindex 647e6952e07..475b3ca2813 100644--- a/test/metabase/lib/test_util/notebook_helpers.cljc+++ b/test/metabase/lib/test_util/notebook_helpers.cljc@@ -9,7 +9,8 @@[metabase.lib.metadata :as lib.metadata][metabase.lib.schema :as lib.schema][metabase.lib.schema.metadata :as lib.schema.metadata]- [metabase.util.malli :as mu]))+ [metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr]))(defn match-display-info [query spec item](let [spec (if (string? spec)@@ -22,11 +23,22 @@(or (m/find-first #(match-display-info query table-spec %) tables)(throw (ex-info "Failed to find table" {:table-spec table-spec, :found (map #(lib/display-info query %) tables)})))))+(mr/def ::display-info-pattern+ "A partial [[metabase.lib.metadata.calculation/display-info]] `=?` pattern, whose values may be predicates or regexes."+ [:map {:closed false, ::mr/deliberately-open true, :description "a partial display info =? pattern"}])++(def ^:private DisplayInfoSpec+ "A `group-spec` or `column-spec` for [[match-display-info]]: either the expected `:display-name`, or a partial+ [[metabase.lib.metadata.calculation/display-info]] to match against."+ [:or+ :string+ ::display-info-pattern])+(mu/defn find-col-with-spec :- ::lib.schema.metadata/column[query :- ::lib.schema/querycolumns :- [:sequential {:min 1} ::lib.schema.metadata/column]- group-spec- column-spec]+ group-spec :- DisplayInfoSpec+ column-spec :- DisplayInfoSpec](let [groups (or (not-empty (lib/group-columns columns))(throw (ex-info "lib/group-columns unexpectedly returned no groups"{:columns columns})))diff --git a/test/metabase/lib_metric/ast/build_test.cljc b/test/metabase/lib_metric/ast/build_test.cljcindex 1efaf32721f..6998d843791 100644--- a/test/metabase/lib_metric/ast/build_test.cljc+++ b/test/metabase/lib_metric/ast/build_test.cljc@@ -34,6 +34,8 @@{:lib/type :metadata/metric:id 42:name "Total Revenue"+ :type :metric+ :database-id (meta/id):dimensions (sample-dimensions):dimension-mappings (sample-mappings):dataset-query (sample-metric-query)})@@ -46,6 +48,7 @@{:lib/type :metadata/measure:id 99:name "Order Count"+ :table-id (meta/id :orders):dimensions (sample-dimensions):dimension-mappings (sample-mappings):definition (sample-measure-query)})diff --git a/test/metabase/lib_metric/ast/compile_test.cljc b/test/metabase/lib_metric/ast/compile_test.cljcindex a75d19ddef5..d0b878b462a 100644--- a/test/metabase/lib_metric/ast/compile_test.cljc+++ b/test/metabase/lib_metric/ast/compile_test.cljc@@ -18,7 +18,15 @@:name "Test Metric":aggregation {:node/type :aggregation/count}:base-table {:node/type :ast/table :id 100}- :metadata {:dataset-query {:database 1}}}+ :metadata {:lib/type :metadata/metric+ :id 42+ :name "Test Metric"+ :type :metric+ :database-id 1+ :dataset-query {:lib/type :mbql/query+ :database 1+ :stages [{:lib/type :mbql.stage/mbql+ :source-table 100}]}}}:dimensions [{:node/type :ast/dimension:id uuid-1:name "category"@@ -63,7 +71,11 @@;;; -------------------------------------------------- Database ID Resolution --------------------------------------------------(deftest ^:parallel compile-missing-database-id-throws-test- (let [ast-no-db (assoc-in sample-ast [:source :metadata] {:dataset-query {}})]+ (let [ast-no-db (assoc-in sample-ast [:source :metadata] {:lib/type :metadata/measure+ :id 1+ :name "Measure without a definition"+ :table-id 100+ :definition nil})](testing "throws when metadata has no database ID"(is (thrown-with-msg? #?(:clj clojure.lang.ExceptionInfo :cljs ExceptionInfo)#"Cannot determine database ID"@@ -120,7 +132,8 @@(is (= 10 (nth (nth agg 2) 2)))))(deftest ^:parallel compile-mbql-aggregation-test- (let [raw-clause [:custom-agg {} [:field {} 10]]+ (let [raw-clause [:sum {:lib/uuid "550e8400-e29b-41d4-a716-446655440003"}+ [:field {:lib/uuid "550e8400-e29b-41d4-a716-446655440004"} 10]]ast-with-mbql (assoc-in sample-ast [:source :aggregation]{:node/type :aggregation/mbql:clause raw-clause})@@ -535,8 +548,11 @@(def ^:private sample-join{:lib/type :mbql/join:alias "Products"- :conditions [[:= {} [:field {} 10] [:field {:join-alias "Products"} 20]]]- :source-table 200+ :conditions [[:=+ {:lib/uuid "550e8400-e29b-41d4-a716-446655440005"}+ [:field {:lib/uuid "550e8400-e29b-41d4-a716-446655440006"} 10]+ [:field {:lib/uuid "550e8400-e29b-41d4-a716-446655440007", :join-alias "Products"} 20]]]+ :stages [{:lib/type :mbql.stage/mbql, :source-table 200}]:fields :all})(def ^:private ast-with-joins@@ -581,7 +597,7 @@stage-0-join (first (:joins (first (:stages result))))](is (= :all (:fields stage-0-join)))))(testing "join with explicit :fields is overridden to :all (dimension system advertises all joined columns)"- (let [join-with-fields (assoc sample-join :fields [[:field {:join-alias "Products"} 20]])+ (let [join-with-fields (assoc sample-join :fields [[:field {:lib/uuid "550e8400-e29b-41d4-a716-446655440008", :join-alias "Products"} 20]])ast (assoc-in sample-ast [:source :joins][{:node/type :ast/join:mbql-join join-with-fields}])@@ -591,7 +607,10 @@(deftest ^:parallel compile-two-stage-filter-separation-test(let [source-filter {:node/type :filter/mbql- :clause [:= {} [:field {} 30] "active"]}+ :clause [:=+ {:lib/uuid "550e8400-e29b-41d4-a716-446655440009"}+ [:field {:lib/uuid "550e8400-e29b-41d4-a716-44665544000a"} 30]+ "active"]}user-filter {:node/type :filter/comparison:operator :=:dimension dim-ref-1diff --git a/test/metabase/lib_metric/ast/plan_test.cljc b/test/metabase/lib_metric/ast/plan_test.cljcindex 9e6f6380d92..b6bb299705c 100644--- a/test/metabase/lib_metric/ast/plan_test.cljc+++ b/test/metabase/lib_metric/ast/plan_test.cljc@@ -21,7 +21,15 @@:name "test":aggregation {:node/type :aggregation/count}:base-table {:node/type :ast/table :id 1}- :metadata {:dataset-query {:database 1}}}+ :metadata {:lib/type :metadata/metric+ :id 1+ :name "test"+ :type :metric+ :database-id 1+ :dataset-query {:lib/type :mbql/query+ :database 1+ :stages [{:lib/type :mbql.stage/mbql+ :source-table 1}]}}}:dimensions (mapv (fn [dim-id] {:node/type :ast/dimension :id dim-id}) group-by-dim-ids):mappings (mapv (fn [dim-id]{:node/type :ast/dimension-mappingdiff --git a/test/metabase/lib_metric/ast/schema_test.cljc b/test/metabase/lib_metric/ast/schema_test.cljcindex 1d4cedcaead..006b0588193 100644--- a/test/metabase/lib_metric/ast/schema_test.cljc+++ b/test/metabase/lib_metric/ast/schema_test.cljc@@ -120,7 +120,7 @@{:node/type :aggregation/distinct:column {:node/type :ast/column :id 1}}{:node/type :aggregation/mbql- :clause [:sum {} [:field {} 1]]}))+ :clause [:sum {:lib/uuid "00000000-0000-0000-0000-000000000001"} [:field {:lib/uuid "00000000-0000-0000-0000-000000000002"} 1]]}))(testing "invalid aggregation nodes"(testing "sum without column"(is (some? (me/humanize (mr/explain ::ast.schema/aggregation-nodediff --git a/test/metabase/lib_metric/definition_test.cljc b/test/metabase/lib_metric/definition_test.cljcindex eda3d9cbe54..cdd8cd9cb41 100644--- a/test/metabase/lib_metric/definition_test.cljc+++ b/test/metabase/lib_metric/definition_test.cljc@@ -6,6 +6,8 @@[metabase.lib-metric.measures :as lib-metric.measures][metabase.lib-metric.metrics :as lib-metric.metrics][metabase.lib-metric.schema :as lib-metric.schema]+ [metabase.lib.core :as lib]+ [metabase.lib.test-metadata :as meta][metabase.util.malli.registry :as mr]));; Ensure multimethod implementations are loaded@@ -32,6 +34,8 @@{:lib/type :metadata/metric:id 42:name "Total Revenue"+ :type :metric+ :database-id 1:dimensions sample-dimensions:dimension-mappings sample-mappings:dataset-query {:database 1 :type :query :query {:source-table 1}}})@@ -40,13 +44,14 @@{:lib/type :metadata/measure:id 99:name "Average Order Value"+ :table-id 2:dimensions sample-dimensions:dimension-mappings sample-mappings- :definition {:database 1 :type :query :query {:source-table 2}}})+ :definition (lib/query meta/metadata-provider (meta/table-metadata :orders))})(def ^:private mock-provider- "A mock metadata provider for testing."- :mock-provider)+ "A metadata provider for testing."+ meta/metadata-provider);;; -------------------------------------------------- from-metric-metadata --------------------------------------------------diff --git a/test/metabase/lib_metric/dimension_test.cljc b/test/metabase/lib_metric/dimension_test.cljcindex a4ebfa9b4e8..f6e515cd4fc 100644--- a/test/metabase/lib_metric/dimension_test.cljc+++ b/test/metabase/lib_metric/dimension_test.cljc@@ -261,8 +261,8 @@(deftest ^:parallel dimensions-changed?-ignores-extra-keys-test(is (not (lib-metric.dimension/dimensions-changed?- [{:id uuid-1 :name "col1" :status :status/active :lib/source :source/table}]- [{:id uuid-1 :name "col1" :status :status/active :lib/source :source/other}]))))+ [{:id uuid-1 :name "col1" :status :status/active :lib/source :source/table-defaults}]+ [{:id uuid-1 :name "col1" :status :status/active :lib/source :source/card}]))))(deftest ^:parallel dimensions-changed?-handles-nil-old-dimensions-test(is (lib-metric.dimension/dimensions-changed?@@ -317,79 +317,79 @@(deftest ^:parallel get-dimension-or-throw-test(testing "finds dimension by id"- (let [dimensions [{:id "dim-1" :name "Dimension 1"}- {:id "dim-2" :name "Dimension 2"}]]- (is (= {:id "dim-1" :name "Dimension 1"}- (lib-metric.dimension/get-dimension-or-throw dimensions "dim-1")))- (is (= {:id "dim-2" :name "Dimension 2"}- (lib-metric.dimension/get-dimension-or-throw dimensions "dim-2")))))+ (let [dimensions [{:id uuid-1 :name "Dimension 1"}+ {:id uuid-2 :name "Dimension 2"}]]+ (is (= {:id uuid-1 :name "Dimension 1"}+ (lib-metric.dimension/get-dimension-or-throw dimensions uuid-1)))+ (is (= {:id uuid-2 :name "Dimension 2"}+ (lib-metric.dimension/get-dimension-or-throw dimensions uuid-2)))))(testing "throws for missing dimension"- (let [dimensions [{:id "dim-1" :name "Dimension 1"}]]+ (let [dimensions [{:id uuid-1 :name "Dimension 1"}]](is (thrown-with-msg? #?(:clj clojure.lang.ExceptionInfo :cljs js/Error)#"Dimension not found"(lib-metric.dimension/get-dimension-or-throw dimensions "nonexistent"))))))(deftest ^:parallel get-dimension-mapping-or-throw-test(testing "finds mapping by dimension id"- (let [mappings [{:dimension-id "dim-1" :target [:field {} 123]}- {:dimension-id "dim-2" :target [:field {} 456]}]]- (is (= {:dimension-id "dim-1" :target [:field {} 123]}- (lib-metric.dimension/get-dimension-mapping-or-throw mappings "dim-1")))- (is (= {:dimension-id "dim-2" :target [:field {} 456]}- (lib-metric.dimension/get-dimension-mapping-or-throw mappings "dim-2")))))+ (let [mappings [{:type :table :dimension-id uuid-1 :target [:field {:lib/uuid "dddddddd-dddd-dddd-dddd-dddddddddddd"} 123]}+ {:type :table :dimension-id uuid-2 :target [:field {:lib/uuid "dddddddd-dddd-dddd-dddd-dddddddddddd"} 456]}]]+ (is (= {:type :table :dimension-id uuid-1 :target [:field {:lib/uuid "dddddddd-dddd-dddd-dddd-dddddddddddd"} 123]}+ (lib-metric.dimension/get-dimension-mapping-or-throw mappings uuid-1)))+ (is (= {:type :table :dimension-id uuid-2 :target [:field {:lib/uuid "dddddddd-dddd-dddd-dddd-dddddddddddd"} 456]}+ (lib-metric.dimension/get-dimension-mapping-or-throw mappings uuid-2)))))(testing "throws for missing mapping"- (let [mappings [{:dimension-id "dim-1" :target [:field {} 123]}]]+ (let [mappings [{:type :table :dimension-id uuid-1 :target [:field {:lib/uuid "dddddddd-dddd-dddd-dddd-dddddddddddd"} 123]}]](is (thrown-with-msg? #?(:clj clojure.lang.ExceptionInfo :cljs js/Error)#"Dimension mapping not found"(lib-metric.dimension/get-dimension-mapping-or-throw mappings "nonexistent"))))))(deftest ^:parallel dimension-target->field-id-test(testing "extracts field ID from field ref"- (is (= 123 (lib-metric.dimension/dimension-target->field-id [:field {} 123])))+ (is (= 123 (lib-metric.dimension/dimension-target->field-id [:field {:lib/uuid "dddddddd-dddd-dddd-dddd-dddddddddddd"} 123])))(is (= 456 (lib-metric.dimension/dimension-target->field-id [:field {:source-field 789} 456]))))(testing "returns nil for non-field refs"(is (nil? (lib-metric.dimension/dimension-target->field-id nil)))(is (nil? (lib-metric.dimension/dimension-target->field-id [:expression {} "foo"]))))(testing "returns nil for field refs with string names instead of IDs"- (is (nil? (lib-metric.dimension/dimension-target->field-id [:field {} "field_name"])))))+ (is (nil? (lib-metric.dimension/dimension-target->field-id [:field {:lib/uuid "dddddddd-dddd-dddd-dddd-dddddddddddd"} "field_name"])))))(deftest ^:parallel resolve-dimension-to-field-id-test(testing "resolves active dimension to field ID"- (let [dimensions [{:id "dim-1" :name "Dimension 1" :status "status/active"}]- mappings [{:dimension-id "dim-1" :target [:field {} 123]}]]- (is (= 123 (lib-metric.dimension/resolve-dimension-to-field-id dimensions mappings "dim-1"))))))+ (let [dimensions [{:id uuid-1 :name "Dimension 1" :status "status/active"}]+ mappings [{:type :table :dimension-id uuid-1 :target [:field {:lib/uuid "dddddddd-dddd-dddd-dddd-dddddddddddd"} 123]}]]+ (is (= 123 (lib-metric.dimension/resolve-dimension-to-field-id dimensions mappings uuid-1))))))(deftest ^:parallel resolve-dimension-to-field-id-throws-for-orphaned-test(testing "throws for orphaned dimension"- (let [dimensions [{:id "dim-1" :name "Dimension 1" :status :status/orphaned}]- mappings [{:dimension-id "dim-1" :target [:field {} 123]}]]+ (let [dimensions [{:id uuid-1 :name "Dimension 1" :status :status/orphaned}]+ mappings [{:type :table :dimension-id uuid-1 :target [:field {:lib/uuid "dddddddd-dddd-dddd-dddd-dddddddddddd"} 123]}]](is (thrown-with-msg? #?(:clj clojure.lang.ExceptionInfo :cljs js/Error)#"Cannot use orphaned dimension"- (lib-metric.dimension/resolve-dimension-to-field-id dimensions mappings "dim-1"))))))+ (lib-metric.dimension/resolve-dimension-to-field-id dimensions mappings uuid-1))))))(deftest ^:parallel resolve-dimension-to-field-id-throws-for-missing-dimension-test(testing "throws for missing dimension"(let [dimensions []- mappings [{:dimension-id "dim-1" :target [:field {} 123]}]]+ mappings [{:type :table :dimension-id uuid-1 :target [:field {:lib/uuid "dddddddd-dddd-dddd-dddd-dddddddddddd"} 123]}]](is (thrown-with-msg? #?(:clj clojure.lang.ExceptionInfo :cljs js/Error)#"Dimension not found"- (lib-metric.dimension/resolve-dimension-to-field-id dimensions mappings "dim-1"))))))+ (lib-metric.dimension/resolve-dimension-to-field-id dimensions mappings uuid-1))))))(deftest ^:parallel resolve-dimension-to-field-id-throws-for-missing-mapping-test(testing "throws for missing mapping"- (let [dimensions [{:id "dim-1" :name "Dimension 1" :status "status/active"}]+ (let [dimensions [{:id uuid-1 :name "Dimension 1" :status "status/active"}]mappings []](is (thrown-with-msg? #?(:clj clojure.lang.ExceptionInfo :cljs js/Error)#"Dimension mapping not found"- (lib-metric.dimension/resolve-dimension-to-field-id dimensions mappings "dim-1"))))))+ (lib-metric.dimension/resolve-dimension-to-field-id dimensions mappings uuid-1))))))(deftest ^:parallel resolve-dimension-to-field-id-throws-for-unresolvable-target-test(testing "throws when target cannot be resolved to field ID"- (let [dimensions [{:id "dim-1" :name "Dimension 1" :status "status/active"}]- mappings [{:dimension-id "dim-1" :target [:field {} "field_name"]}]]+ (let [dimensions [{:id uuid-1 :name "Dimension 1" :status "status/active"}]+ mappings [{:type :table :dimension-id uuid-1 :target [:field {:lib/uuid "dddddddd-dddd-dddd-dddd-dddddddddddd", :base-type :type/Text} "field_name"]}]](is (thrown-with-msg? #?(:clj clojure.lang.ExceptionInfo :cljs js/Error)#"Cannot resolve dimension target to field ID"- (lib-metric.dimension/resolve-dimension-to-field-id dimensions mappings "dim-1"))))))+ (lib-metric.dimension/resolve-dimension-to-field-id dimensions mappings uuid-1))))));;; ----------------------------------------- effective-type / base-type fallback -----------------------------------------diff --git a/test/metabase/lib_metric/filter_test.cljc b/test/metabase/lib_metric/filter_test.cljcindex 3e506256d11..eb95e88bb2a 100644--- a/test/metabase/lib_metric/filter_test.cljc+++ b/test/metabase/lib_metric/filter_test.cljc@@ -811,13 +811,13 @@{:lib/type :metadata/metric, :id 2, :name "orders", :display-name "Orders", :table-id 30})(def ^:private seg-source-segment- {:lib/type :metadata/segment, :id 100, :name "Big Orders", :table-id 10, :definition {}})+ {:lib/type :metadata/segment, :id 100, :name "Big Orders", :table-id 10, :definition nil})(def ^:private seg-joined-segment- {:lib/type :metadata/segment, :id 200, :name "Active Users", :table-id 20, :definition {}})+ {:lib/type :metadata/segment, :id 200, :name "Active Users", :table-id 20, :definition nil})(def ^:private seg-second-source-segment- {:lib/type :metadata/segment, :id 300, :name "Live Widgets", :table-id 30, :definition {}})+ {:lib/type :metadata/segment, :id 300, :name "Live Widgets", :table-id 30, :definition nil})(defn- seg-metric-fetcher [{:keys [id]}](let [wanted (or id #{1 2})]diff --git a/test/metabase/metabot/agent/core_test.clj b/test/metabase/metabot/agent/core_test.cljindex 0a23a50db8c..98176bea0dd 100644--- a/test/metabase/metabot/agent/core_test.clj+++ b/test/metabase/metabot/agent/core_test.clj@@ -206,7 +206,7 @@:profile-id profile-id:context {:capabilities ["permission:write_sql_queries"]:user_is_viewing [{:type "code_editor"- :buffers [{:id "buf-1"}]}]}}))]+ :buffers [{:id "buf-1" :source {:language "sql" :database_id nil} :cursor {:line 0 :column 0}}]}]}}))]{:llm-calls @call-count :parts parts})))))(deftest permission-denial-ends-a-forced-tool-call-turn-testdiff --git a/test/metabase/metabot/agent/messages_test.clj b/test/metabase/metabot/agent/messages_test.cljindex 9733dad8aa2..f0fb328be9d 100644--- a/test/metabase/metabot/agent/messages_test.clj+++ b/test/metabase/metabot/agent/messages_test.clj@@ -26,7 +26,7 @@(messages/input-message->parts {:role :assistant :content "Hi there"}))))(testing "assistant with tool_calls (OpenAI style)"(is (=? [{:type :text :text "Searching..."}- {:type :tool-input :id "t1" :function "search" :arguments {:q "test"}}]+ {:type :tool-input :id "t1" :function "search" :arguments {"q" "test"}}](messages/input-message->parts{:role :assistant:content "Searching..."diff --git a/test/metabase/metabot/api_test.clj b/test/metabase/metabot/api_test.cljindex 9c37fe2bccd..69267b68553 100644--- a/test/metabase/metabot/api_test.clj+++ b/test/metabase/metabot/api_test.clj@@ -850,7 +850,7 @@turn-state (conj {:type :data :data-type "state" :data turn-state}))))](mt/with-model-cleanup [:model/MetabotMessage [:model/MetabotConversation :created_at]](let [conversation-id (str (random-uuid))- turn-1-state {:queries {:q_1 {:database 1}} :todos [{:id "a" :status "pending"}]}+ turn-1-state {:queries {"q_1" {:database 1}} :todos [{:id "a" :status "pending"}]}first-response (mt/user-http-request :rasta :post 202 "metabot/agent-streaming"{:message "make a query":context {}@@ -867,7 +867,7 @@(is (= {} (first @seeded-states))"a new conversation seeds the loop with empty state")(is (= turn-1-state (second @seeded-states))- "the follow-up turn is seeded from the DB partial, keywordized — no client echo")+ "the follow-up turn is seeded from the DB partial, normalized — no client echo")(mt/user-http-request :rasta :post 202 "metabot/agent-streaming"{:message "another":context {}@@ -957,13 +957,13 @@(testing "metabase/ provider prefix sets ai_proxied true and stores bare model names"(let [msg (start-and-finalize-with-provider! "metabase/anthropic/claude-sonnet-4-6")](is (true? (:ai_proxied msg)))- (is (= {:claude-sonnet-4-6 {:prompt 100 :completion 50}}+ (is (= {"claude-sonnet-4-6" {:prompt 100 :completion 50}}(:usage msg))"usage keys should be bare model names, not metabase/anthropic/...")))(testing "BYOK provider (no metabase/ prefix) sets ai_proxied false"(let [msg (start-and-finalize-with-provider! "anthropic/claude-sonnet-4-6")](is (false? (:ai_proxied msg)))- (is (= {:claude-sonnet-4-6 {:prompt 100 :completion 50}}+ (is (= {"claude-sonnet-4-6" {:prompt 100 :completion 50}}(:usage msg))))))(deftest finalize-assistant-turn-data-part-filtering-test@@ -986,10 +986,10 @@{:type :data :data-type "transform_suggestion" :version 1 :data {}}{:type :data :data-type "adhoc_viz" :version 1 :data {:query {} :link "/q"}}{:type :data :data-type "static_viz" :version 1 :data {:entity_id 1}}- {:type :data :data-type "state" :data {:step 1}}+ {:type :data :data-type "state" :data {:todos [{:id "1"}]}}{:type :usage :model "claude-sonnet-4-6" :usage {:promptTokens 1 :completionTokens 1}}{:type :finish}]- :turn-state {:step 1})+ :turn-state {:todos [{:id "1"}]})(let [msg (t2/select-one :model/MetabotMessage assistant-msg-id)part-types (into #{} (map :type) (:data msg))data-types (into #{}@@ -1002,7 +1002,7 @@"text parts survive")(is (not-any? part-types #{"start" "usage" "finish"})"stream metadata is dropped")- (is (= {:step 1} (:state msg))+ (is (= {:todos [{:id "1"}]} (:state msg))"the turn's partial state lands on the message row"))))(finally(t2/delete! :model/MetabotMessage :conversation_id conv-id)@@ -1080,16 +1080,21 @@:result {:structured-output {:some "data"}}}])))))(defn- legacy-query- "A legacy inner-query-style map suitable for [[#'api/upgrade-viewing-queries]]."+ "A legacy inner-query-style map suitable for [[upgrade-viewing-queries]]."[]{:database (mt/id):query {:source-table (mt/id :orders)}:type :query})+(defn- upgrade-viewing-queries+ "Decodes `items` the way the `/agent-streaming` endpoint does, then runs [[api/upgrade-viewing-queries]] on them."+ [items]+ (#'api/upgrade-viewing-queries (lib/normalize [:vector metabot.context/ViewingItemSchema] items)))+(deftest upgrade-viewing-queries-upgradable-types-test(doseq [item-type ["adhoc" "question" "metric" "model"]](testing (str "upgrades query for type=" item-type)- (let [result (#'api/upgrade-viewing-queries [{:type item-type :query (legacy-query)}])+ (let [result (upgrade-viewing-queries [{:type item-type :query (legacy-query)}])q (:query (first result))](is (= :mbql/query (:lib/type q)))(is (= (mt/id) (:database q)))))))@@ -1100,7 +1105,7 @@:query lq:chart_configs [{:query lq}{:query lq}]}- result (first (#'api/upgrade-viewing-queries [item]))]+ result (first (upgrade-viewing-queries [item]))](is (= :mbql/query (:lib/type (:query result))))(is (every? #(= :mbql/query (:lib/type (:query %)))(:chart_configs result)))))@@ -1108,9 +1113,9 @@(deftest upgrade-viewing-queries-missing-keys-test(testing "items without :query are unchanged"(let [item {:type "adhoc"}]- (is (= [item] (#'api/upgrade-viewing-queries [item])))))+ (is (= [item] (upgrade-viewing-queries [item])))))(testing "items without :chart_configs keep no chart_configs"- (let [result (first (#'api/upgrade-viewing-queries [{:type "question" :query (legacy-query)}]))]+ (let [result (first (upgrade-viewing-queries [{:type "question" :query (legacy-query)}]))](is (nil? (:chart_configs result))))))(deftest upgrade-viewing-queries-mixed-items-test@@ -1118,7 +1123,7 @@items [{:type "adhoc" :query lq}{:type "dashboard"}{:type "model" :query lq :chart_configs [{:query lq}]}]- result (#'api/upgrade-viewing-queries items)]+ result (upgrade-viewing-queries items)](is (=? [{:query {:lib/type :mbql/query}}{}{:query {:lib/type :mbql/query}@@ -1131,7 +1136,7 @@items [{:type "adhoc" :query q}{:type "dashboard"}{:type "model" :query q :chart_configs [{:query q}]}]- result (#'api/upgrade-viewing-queries items)]+ result (upgrade-viewing-queries items)](is (=? [{:type "adhoc" :query q}{:type "dashboard"}{:type "model" :query q :chart_configs [{:query q}]}]@@ -1145,7 +1150,7 @@items [{:type "adhoc" :query legacy}{:type "dashboard"}{:type "model" :query legacy :chart_configs [{:query legacy}]}]- result (#'api/upgrade-viewing-queries items)]+ result (upgrade-viewing-queries items)](is (=? [{:type "adhoc" :query native}{:type "dashboard"}{:type "model" :query native :chart_configs [{:query native}]}]diff --git a/test/metabase/metabot/persistence_test.clj b/test/metabase/metabot/persistence_test.cljindex 5e5e89f2ee7..2f355493d05 100644--- a/test/metabase/metabot/persistence_test.clj+++ b/test/metabase/metabot/persistence_test.clj@@ -1516,7 +1516,7 @@msg-id (seed-turn! conversation-id "hi"[{:type :text :text "reply"}]:turn-state {:queries {"q2" {:database 2}}})]- (is (= {:queries {:q2 {:database 2}}}+ (is (= {:queries {"q2" {:database 2}}}(t2/select-one-fn :state :model/MetabotMessage msg-id)))(is (nil? (t2/select-one-fn :state :model/MetabotConversation :id conversation-id)))))))diff --git a/test/metabase/metabot/schema/v2_test.clj b/test/metabase/metabot/schema/v2_test.cljindex 1a80e69dc92..a881a933e31 100644--- a/test/metabase/metabot/schema/v2_test.clj+++ b/test/metabase/metabot/schema/v2_test.clj@@ -7,7 +7,7 @@(def ^:private chunks"A representative chunk for each uiMessageChunkSchema union member."[{:type "text-start" :id "t1"}- {:type "text-delta" :id "t1" :delta "Hi" :providerMetadata {:openai {:x 1}}}+ {:type "text-delta" :id "t1" :delta "Hi" :providerMetadata {:openai {:itemId "rs_1"}}}{:type "text-end" :id "t1"}{:type "error" :errorText "boom"}{:type "tool-input-start" :toolCallId "tc1" :toolName "search" :dynamic false}@@ -47,7 +47,7 @@{:type "tool-search" :toolCallId "tc1" :state "approval-responded" :input {:q "x"}:approval {:id "ap1" :approved true :reason "ok"}}{:type "tool-search" :toolCallId "tc1" :state "output-available" :input {:q "x"}- :output {:rows []} :preliminary false}+ :output {:output "No rows."} :preliminary false}{:type "tool-search" :toolCallId "tc1" :state "output-error" :input {:q "x"}:rawInput "{\"q\"" :errorText "failed"}{:type "tool-search" :toolCallId "tc1" :state "output-denied" :input {:q "x"}@@ -67,9 +67,9 @@(is (nil? (mr/explain [:sequential ::schema.v2/ui-message-part] ui-message-parts))))(deftest ^:parallel aisdk-runtime-validator-equivalence-test- (testing "at-rest parts are open maps: the upstream validator strips undeclared keys rather than rejecting"- (is (mr/validate ::schema.v2/ui-message-part- {:type "text" :text "hi" :somethingExtra 1})))+ (testing "at-rest parts are closed maps: only the keys persistence writes are accepted"+ (is (not (mr/validate ::schema.v2/ui-message-part+ {:type "text" :text "hi" :somethingExtra 1}))))(testing "wire chunks are closed maps: the upstream validator rejects undeclared keys"(is (not (mr/validate ::schema.v2/ui-message-chunk{:type "text-start" :id "t1" :somethingExtra 1}))))@@ -97,7 +97,7 @@(is (not (mr/validate ::schema.v2/ui-message-part (update part :approval dissoc :approved))))))(testing "output-available approvals must be approved"(let [part {:type "tool-search" :toolCallId "tc1" :state "output-available"- :input {:q "x"} :output {:rows []} :approval {:id "ap1" :approved true}}]+ :input {:q "x"} :output {:output "No rows."} :approval {:id "ap1" :approved true}}](is (mr/validate ::schema.v2/ui-message-part part))(is (not (mr/validate ::schema.v2/ui-message-part (assoc-in part [:approval :approved] false))))))(testing "output-denied approvals must be denied"@@ -112,4 +112,4 @@:role "assistant":parts [{:type "text" :text "Hello"}{:type "tool-search" :toolCallId "tc1" :state "output-available"- :input {:q "x"} :output {:rows []}}]}))))+ :input {:q "x"} :output {:output "No rows."}}]}))))diff --git a/test/metabase/metabot/tools/sql_test.clj b/test/metabase/metabot/tools/sql_test.cljindex f1e5ebb9e0e..c5d9e88f6bd 100644--- a/test/metabase/metabot/tools/sql_test.clj+++ b/test/metabase/metabot/tools/sql_test.clj@@ -52,7 +52,7 @@(defn- create-sql-query-in-code-editor[args](binding [shared/*memory-atom* (atom {:context {:user_is_viewing [{:type "code_editor"- :buffers [{:id "buf-1"}]}]}})]+ :buffers [{:id "buf-1" :source {:language "sql" :database_id nil} :cursor {:line 0 :column 0}}]}]}})](agent-sql/create-sql-query-code-edit-tool (merge {:sql_query "SELECT 1":title "Results"}args))))@@ -208,7 +208,7 @@(is (= :native (get-in entity [:query :query :type])))))(testing "an open code-editor buffer wins"(let [parts (:data-parts (run {:user_is_viewing [{:type "code_editor"- :buffers [{:id "buf-1"}]}]}))]+ :buffers [{:id "buf-1" :source {:language "sql" :database_id nil} :cursor {:line 0 :column 0}}]}]}))](is (= 1 (count parts)))(is (= "code_edit" (:data-type (first parts)))))))))))))diff --git a/test/metabase/metabot/tools/util_test.clj b/test/metabase/metabot/tools/util_test.cljindex 270a9bb7e1a..b8109eb6c6d 100644--- a/test/metabase/metabot/tools/util_test.clj+++ b/test/metabase/metabot/tools/util_test.clj@@ -275,9 +275,9 @@:model/Table {active-table-id :id} {:db_id db-id, :name "active_table", :active true, :visibility_type nil}:model/Table {inactive-table-id :id} {:db_id db-id, :name "inactive_table", :active false, :visibility_type nil}](mt/with-current-user (mt/user->id :crowberto)- (is (= active-table-id (:id (metabot.tools.util/get-table active-table-id))))+ (is (= active-table-id (:id (metabot.tools.util/get-table active-table-id :db_id))))(is (thrown? clojure.lang.ExceptionInfo- (metabot.tools.util/get-table inactive-table-id)))))))+ (metabot.tools.util/get-table inactive-table-id :db_id)))))))(deftest find-column-by-field-id-test(testing "finds column by integer field ID"diff --git a/test/metabase/metrics/dimension_test.clj b/test/metabase/metrics/dimension_test.cljindex 948c7b48591..e908f9ddfea 100644--- a/test/metabase/metrics/dimension_test.clj+++ b/test/metabase/metrics/dimension_test.clj@@ -87,23 +87,29 @@(is (= :metabase.lib-metric.schema/dimension-id(entry-form ::metrics.dimension/dimension-mapping :dimension-id)))))+(defn- dim-id+ "A dimension UUID ending in `n`."+ [n]+ (format "00000000-0000-0000-0000-%012d" n))+(defn- metric- "A metric-shaped map with one dimension per `[dim-id field-id]` pair."+ "A metric Card with one dimension per `[dim-id field-id]` pair."[& dim-id+field-id]{:dimensions (for [[dim-id _] dim-id+field-id]{:id dim-id}):dimension_mappings (for [[dim-id field-id] dim-id+field-id]- {:dimension-id dim-id- :target [:field {} field-id]})})+ {:type :table+ :dimension-id dim-id+ :target [:field {:lib/uuid (str (random-uuid))} field-id]})})(deftest annotate-dimensions-resolves-each-dimension-once-test(testing "annotate-dimensions-with-field-data resolves each dimension's field-id exactly once";; It used to resolve once to collect the field-ids to batch-select, then again per dimension;; while merging the rows back on — double the work, and `resolve-dimension-to-field-id` walks;; the mapping list for every call.- (let [metrics [(metric ["d1" (mt/id :venues :price)]- ["d2" (mt/id :venues :name)])- (metric ["d3" (mt/id :orders :total)])]+ (let [metrics [(metric [(dim-id 1) (mt/id :venues :price)]+ [(dim-id 2) (mt/id :venues :name)])+ (metric [(dim-id 3) (mt/id :orders :total)])]calls (atom 0)real-resolve lib-metric/resolve-dimension-to-field-id](with-redefs [lib-metric/resolve-dimension-to-field-id@@ -122,18 +128,17 @@:id [:in [price-id name-id]])[m] (metrics.dimension/annotate-dimensions-with-field-data[:description]- [(metric ["d1" price-id] ["d2" name-id])])]+ [(metric [(dim-id 1) price-id] [(dim-id 2) name-id])])](is (= [(get expected price-id) (get expected name-id)](map :description (:dimensions m))))- (is (= ["d1" "d2"] (map :id (:dimensions m)))+ (is (= [(dim-id 1) (dim-id 2)] (map :id (:dimensions m)))"dimension order and identity are preserved"))))(deftest annotate-dimensions-nils-unresolvable-dimensions-test(testing "a dimension whose field-id can't be resolved still gets the columns, as nil"(let [[m] (metrics.dimension/annotate-dimensions-with-field-data[:description]- ;; no mapping for d1 → resolve throws- [{:dimensions [{:id "d1"}] :dimension_mappings []}])]+ [{:dimensions [{:id (dim-id 1)}] :dimension_mappings []}])](is (= [nil] (map :description (:dimensions m))))(is (contains? (first (:dimensions m)) :description)"the column key is present even when unresolvable"))))@@ -144,9 +149,9 @@[msgs [metabase.metrics.dimension :debug]](metrics.dimension/annotate-dimensions-with-field-data[:description]- [{:dimensions [{:id "d1"}] :dimension_mappings []}])+ [{:dimensions [{:id (dim-id 1)}] :dimension_mappings []}])(msgs))]- (is (some #(str/includes? (str (:message %)) "d1") msgs)+ (is (some #(str/includes? (str (:message %)) (dim-id 1)) msgs)"the failing dimension id should appear in the log"))))(deftest annotate-dimensions-passes-through-metrics-without-dimensions-testdiff --git a/test/metabase/model_persistence/task/persist_refresh_test.clj b/test/metabase/model_persistence/task/persist_refresh_test.cljindex 340d06daeb3..3b0bd0138ac 100644--- a/test/metabase/model_persistence/task/persist_refresh_test.clj+++ b/test/metabase/model_persistence/task/persist_refresh_test.clj@@ -273,9 +273,9 @@(let [task-name (mt/random-name)](#'task.persist-refresh/save-task-history! task-name (mt/id)(fn []- {:foo "bar"}))+ {:success 1 :error 0}))(is (=? {:task task-name- :task_details {:foo "bar"}+ :task_details {:success 1 :error 0}:status :success}(t2/select-one :model/TaskHistory :task task-name)))))))@@ -285,9 +285,11 @@(let [task-name (mt/random-name)](#'task.persist-refresh/save-task-history! task-name (mt/id)(fn []- {:error-details ["some-error"]}))+ {:success 0+ :error 1+ :error-details [{:persisted-info-id 1 :error "some-error"}]}))(is (=? {:task task-name- :task_details {:error-details ["some-error"]}+ :task_details {:error-details [{:persisted-info-id 1 :error "some-error"}]}:status :failed}(t2/select-one :model/TaskHistory :task task-name)))))))@@ -300,7 +302,9 @@(reset! email-sent true))](#'task.persist-refresh/save-task-history! "persist-refresh" (mt/id)(fn []- {:error-details ["some-error"]}))+ {:success 0+ :error 1+ :error-details [{:persisted-info-id 1 :error "some-error"}]}))(is (true? @email-sent)))))))(deftest persisted-model-refresh-error-event-accepts-quartz-trigger-testdiff --git a/test/metabase/models/util/spec_update_test.clj b/test/metabase/models/util/spec_update_test.cljindex b42acf4b188..5d8cde875d3 100644--- a/test/metabase/models/util/spec_update_test.clj+++ b/test/metabase/models/util/spec_update_test.clj@@ -30,9 +30,9 @@"spec for testing."{:model :root:compare-cols [:name]- :nested-specs {:foo {:model :foo+ :nested-specs [[:foo {:model :foo:compare-cols [:name]- :fk-column :root_id}}})+ :fk-column :root_id}]]})(deftest basic-create-test(testing "Creating a new record with no existing data"@@ -141,11 +141,11 @@{:model :root:compare-cols [:name]:extra-cols [:unrelated]- :nested-specs {:bars {:model :bar- :multi-row? true- :fk-column :root_id- :extra-cols [:unrelated]- :compare-cols [:name]}}})+ :nested-specs [[:bars {:model :bar+ :multi-row? true+ :fk-column :root_id+ :extra-cols [:unrelated]+ :compare-cols [:name]}]]})(deftest multi-row-create-test(testing "Creating root with nested multi-row models"@@ -224,12 +224,12 @@:id-col :id:compare-cols [:name]:extra-cols [:unrelated]- :nested-specs {:bars {:model :bar- :id-col :id- :multi-row? true- :fk-column :foo_id- :compare-cols [:name]- :extra-cols [:unrelated]}}}+ :nested-specs [[:bars {:model :bar+ :id-col :id+ :multi-row? true+ :fk-column :foo_id+ :compare-cols [:name]+ :extra-cols [:unrelated]}]]}existing-data {:id 1:name "Test":unrelated "abc"@@ -265,20 +265,20 @@{:model :foo:id-col :uuid:compare-cols [:name]- :nested-specs {:bars {:model :bar+ :nested-specs [[:bars {:model :bar+ :compare-cols [:name]+ :multi-row? true+ :fk-column :foo_id+ :nested-specs [[:quxes {:model :bar_qux+ :compare-cols [:name]+ :multi-row? true+ :fk-column :bar_id}]]}]+ [:qux {:model :qux:compare-cols [:name]- :multi-row? true:fk-column :foo_id- :nested-specs {:quxes {:model :bar_qux- :compare-cols [:name]- :multi-row? true- :fk-column :bar_id}}}- :qux {:model :qux- :compare-cols [:name]- :fk-column :foo_id- :nested-specs {:bar {:model :qux_bar- :compare-cols [:name]- :fk-column :qux_id}}}}})+ :nested-specs [[:bar {:model :qux_bar+ :compare-cols [:name]+ :fk-column :qux_id}]]}]]})(deftest complex-nested-model-test(testing "Complex nested model with multiple levels"@@ -345,14 +345,14 @@"A spec with nested multi-row models"{:model :foo:compare-cols [:name]- :nested-specs {:bars {:model :bar- :compare-cols [:name]- :multi-row? true- :fk-column :foo_id- :nested-specs {:quxes {:model :bar_qux- :compare-cols [:name]- :multi-row? true- :fk-column :bar_id}}}}})+ :nested-specs [[:bars {:model :bar+ :compare-cols [:name]+ :multi-row? true+ :fk-column :foo_id+ :nested-specs [[:quxes {:model :bar_qux+ :compare-cols [:name]+ :multi-row? true+ :fk-column :bar_id}]]}]]})(deftest nested-sequential-test(testing "adding 2 layers of nested sequential updates"@@ -407,9 +407,9 @@"A spec with ref-in-parent references"{:model :parent:compare-cols [:name]- :nested-specs {:child {:model :child- :compare-cols [:name]- :ref-in-parent :child_id}}})+ :nested-specs [[:child {:model :child+ :compare-cols [:name]+ :ref-in-parent :child_id}]]})(deftest ref-in-parent-create-test(testing "Creating parent with referenced child (ref-in-parent)"@@ -487,19 +487,19 @@"A spec with multiple ref-in-parent references"{:model :order:compare-cols [:number]- :nested-specs {:customer {:model :customer- :compare-cols [:name]- :ref-in-parent :customer_id}- :items {:model :item- :multi-row? true- :fk-column :order_id- :compare-cols [:product_name :quantity]}- :payment {:model :payment- :compare-cols [:amount]- :ref-in-parent :payment_id- :nested-specs {:processor {:model :payment_processor- :compare-cols [:name]- :ref-in-parent :processor_id}}}}})+ :nested-specs [[:customer {:model :customer+ :compare-cols [:name]+ :ref-in-parent :customer_id}]+ [:items {:model :item+ :multi-row? true+ :fk-column :order_id+ :compare-cols [:product_name :quantity]}]+ [:payment {:model :payment+ :compare-cols [:amount]+ :ref-in-parent :payment_id+ :nested-specs [[:processor {:model :payment_processor+ :compare-cols [:name]+ :ref-in-parent :processor_id}]]}]]})(deftest complex-ref-in-parent-test(testing "Complex flow with multiple ref-in-parent relationships"@@ -532,13 +532,13 @@"A spec with ref-in-parent inside a multi-row spec"{:model :project:compare-cols [:name]- :nested-specs {:tasks {:model :task- :multi-row? true- :fk-column :project_id- :compare-cols [:description]- :nested-specs {:assignee {:model :user- :compare-cols [:name]- :ref-in-parent :assignee_id}}}}})+ :nested-specs [[:tasks {:model :task+ :multi-row? true+ :fk-column :project_id+ :compare-cols [:description]+ :nested-specs [[:assignee {:model :user+ :compare-cols [:name]+ :ref-in-parent :assignee_id}]]}]]})(deftest multi-row-with-ref-create-test(testing "Creating a project with tasks that have assignee references"diff --git a/test/metabase/notification/payload/impl/system_event_test.clj b/test/metabase/notification/payload/impl/system_event_test.cljindex 6b0af14e1cd..201ecca8673 100644--- a/test/metabase/notification/payload/impl/system_event_test.clj+++ b/test/metabase/notification/payload/impl/system_event_test.clj@@ -195,7 +195,7 @@(mt/with-temporary-setting-values [site-url "https://metabase.com"](-> (notification.tu/with-captured-channel-send!(user/create-and-invite-user! {:first_name "Newbie" :email (mt/random-email)}- {:first_name "Admin" :email "[email protected]"}+ {:id (mt/user->id :crowberto) :first_name "Admin" :email "[email protected]"}falseinvite-target)):channel/email first :message first :content)))]@@ -215,7 +215,7 @@(mt/with-temporary-setting-values [site-url "https://metabase.com"](-> (notification.tu/with-captured-channel-send!(user/create-and-invite-user! {:first_name "Newbie" :email (mt/random-email)}- {:first_name "Admin" :email "[email protected]"}+ {:id (mt/user->id :crowberto) :first_name "Admin" :email "[email protected]"}falseinvite-target)):channel/email first :message first :content)))]@@ -234,7 +234,7 @@site-name "SuperStar"](-> (notification.tu/with-captured-channel-send!(user/create-and-invite-user! {:first_name "Newbie" :email (mt/random-email)}- {:first_name "Ngoc" :email "[email protected]"}+ {:id (mt/user->id :crowberto) :first_name "Ngoc" :email "[email protected]"}falseinvite-target)):channel/email first)))diff --git a/test/metabase/notification/send_test.clj b/test/metabase/notification/send_test.cljindex 5bbb2402308..11ed9754498 100644--- a/test/metabase/notification/send_test.clj+++ b/test/metabase/notification/send_test.clj@@ -48,7 +48,7 @@:channel_id (:id chn-2):recipients [{:type :notification-recipient/user:user_id (mt/user->id :rasta)}]}])- notification-info (assoc n :payload {:event_info {:test true}+ notification-info (assoc n :payload {:event_info {}:event_topic :event/test})expected-notification-payload (mt/malli=?[:mapdiff --git a/test/metabase/parameters/params_test.clj b/test/metabase/parameters/params_test.cljindex 40c657f3e81..d15a09aee88 100644--- a/test/metabase/parameters/params_test.clj+++ b/test/metabase/parameters/params_test.clj@@ -210,19 +210,20 @@(is (not (contains? param-fields "p7"))))))))(deftest ^:parallel card->template-tag-test- (let [card {:dataset_query (mt/native-query {:query "SELECT *"- :template-tags {"id" {:name "id"- :display-name "ID"- :id "11111111"- :type :dimension- :widget-type :number- :dimension [:field (mt/id :venues :id) nil]}- "name" {:name "name"- :display-name "Name"- :id "aaaaaaaa"- :type :dimension- :widget-type :number- :dimension [:field "name" {:base-type :type/Text}]}}})}]+ (let [card {:dataset_query (lib-be/normalize-query+ (mt/native-query {:query "SELECT *"+ :template-tags {"id" {:name "id"+ :display-name "ID"+ :id "11111111"+ :type :dimension+ :widget-type :number+ :dimension [:field (mt/id :venues :id) nil]}+ "name" {:name "name"+ :display-name "Name"+ :id "aaaaaaaa"+ :type :dimension+ :widget-type :number+ :dimension [:field "name" {:base-type :type/Text}]}}}))}](testing "card->template-tag-param-id->field-ids"(is (= {"11111111" #{(mt/id :venues :id)}"aaaaaaaa" #{}}@@ -234,20 +235,21 @@(deftest ^:parallel duplicate-column-names-test(testing "columns with duplicated names get mapped correctly to parameters"(testing "native queries"- (let [card {:dataset_query (mt/native-query {:query "SELECT *"- :template-tags- {"tag1" {:name "tag1"- :display-name "Tag 1"- :id "11111111"- :type :dimension- :widget-type :number- :dimension [:field (mt/id :orders :id) nil]}- "tag2" {:name "tag2"- :display-name "Tag 2"- :id "aaaaaaaa"- :type :dimension- :widget-type :number- :dimension [:field (mt/id :products :id) nil]}}})}]+ (let [card {:dataset_query (lib-be/normalize-query+ (mt/native-query {:query "SELECT *"+ :template-tags+ {"tag1" {:name "tag1"+ :display-name "Tag 1"+ :id "11111111"+ :type :dimension+ :widget-type :number+ :dimension [:field (mt/id :orders :id) nil]}+ "tag2" {:name "tag2"+ :display-name "Tag 2"+ :id "aaaaaaaa"+ :type :dimension+ :widget-type :number+ :dimension [:field (mt/id :products :id) nil]}}}))}](testing "card->template-tag-param-id->field-ids"(is (= {"11111111" #{(mt/id :orders :id)}"aaaaaaaa" #{(mt/id :products :id)}}diff --git a/test/metabase/parameters/schema_test.cljc b/test/metabase/parameters/schema_test.cljcindex cdeaf7b0c40..98db8475e9b 100644--- a/test/metabase/parameters/schema_test.cljc+++ b/test/metabase/parameters/schema_test.cljc@@ -3,26 +3,29 @@#?@(:clj [[metabase.api.macros :as api.macros][metabase.util.json :as json]])[clojure.test :refer [deftest is testing]]+ [metabase.lib.core :as lib][metabase.parameters.schema :as parameters.schema]))(deftest ^:parallel default-to-type-text-test(is (= {:id "x", :type :text}- (parameters.schema/normalize-parameter {:id "x"}))))+ (lib/normalize ::parameters.schema/parameter {:id "x"}))))(deftest ^:parallel normalize-parameter-from-json-test(is (= {:type :text, :id "e7f8ca", :name "variable", :slug "foo_bar", :value 15, :values_source_type :card}- (parameters.schema/normalize-parameter+ (lib/normalize+ ::parameters.schema/parameter{"id" "e7f8ca", "name" "variable", "slug" "foo_bar", "type" "text", "value" 15, "values_source_type" "card"}))))(deftest ^:parallel normalize-parameter-mapping-remove-negative-card-ids(testing "Apparently some FE code accidentally sent `card_id -1` and now we have to work around it"(is (= {:parameter_id "53bb6214", :target [:variable [:template-tag "id"]]}- (parameters.schema/normalize-parameter-mapping+ (lib/normalize+ ::parameters.schema/parameter-mapping{"parameter_id" "53bb6214", "card_id" -1, "target" ["variable" ["template-tag" "id"]]})))))(deftest ^:parallel normalize-parameters-without-adding-default-type-test(is (= [{:id "x", :target [:dimension [:template-tag "y"]]}]- (parameters.schema/normalize-parameters-without-adding-default-types [{"id" "x", "target" ["dimension" ["template-tag" "y"]]}]))))+ (lib/normalize ::parameters.schema/parameters-with-optional-types [{"id" "x", "target" ["dimension" ["template-tag" "y"]]}]))))#?(:clj(deftest api-parameter-values-decode-strips-extra-keys-testdiff --git a/test/metabase/permissions/models/permissions_test.clj b/test/metabase/permissions/models/permissions_test.cljindex 6d1b34c71f9..7a29e7f1a32 100644--- a/test/metabase/permissions/models/permissions_test.clj+++ b/test/metabase/permissions/models/permissions_test.clj@@ -114,20 +114,17 @@(deftest ^:parallel perms-objects-set-for-parent-collection-test(are [input expected] (= expected(apply perms/perms-objects-set-for-parent-collection input))- [{:collection_id 1337} :read] #{"/collection/1337/read/"}- [{:collection_id 1337} :write] #{"/collection/1337/"}- [{:collection_id nil} :read] #{"/collection/root/read/"}- [{:collection_id nil} :write] #{"/collection/root/"})+ [1337 :read] #{"/collection/1337/read/"}+ [1337 :write] #{"/collection/1337/"}+ [nil :read] #{"/collection/root/read/"}+ [nil :write] #{"/collection/root/"})(testing "invalid input"- (doseq [[reason inputs] {"map must have `:collection_id` key"- [[{} :read]]-- "must be a map"- [[100 :read]- [nil :read]]+ (doseq [[reason inputs] {"must be a positive int or nil"+ [[-1 :read]+ ["100" :read]]"read-or-write must be `:read` or `:write`"- [[{:collection_id nil} :readwrite]]}+ [[nil :readwrite]]}input inputs](testing reason(testing (pr-str (cons 'perms-objects-set-for-parent-collection input))diff --git a/test/metabase/permissions/path_test.clj b/test/metabase/permissions/path_test.cljindex eb5aae36af5..59837417991 100644--- a/test/metabase/permissions/path_test.clj+++ b/test/metabase/permissions/path_test.clj@@ -13,8 +13,6 @@(doseq [[input expected]{1 {:read "/collection/1/read/":readwrite "/collection/1/"}- {:id 1} {:read "/collection/1/read/"- :readwrite "/collection/1/"}collection/root-collection {:read "/collection/root/read/":readwrite "/collection/root/"}(assoc collection/root-collection :namespace "snippets") {:read "/collection/namespace/snippets/root/read/"diff --git a/test/metabase/plugins/initialize_test.clj b/test/metabase/plugins/initialize_test.cljindex 69efaa6cec4..edbf44e100e 100644--- a/test/metabase/plugins/initialize_test.clj+++ b/test/metabase/plugins/initialize_test.clj@@ -135,11 +135,11 @@plugin (fn [plugin-name marker]{:metabase-plugin-api-version initialize/plugin-api-version:info {:name plugin-name :version "1.0.0"}- :init [{:step "test" :marker marker}]})]+ :init [{:step "load-namespace" :namespace (name marker)}]})](mt/with-dynamic-fn-redefs [deps/all-dependencies-satisfied? (constantly true)deps/update-unsatisfied-deps! (constantly [])- init-steps/do-init-steps! (fn [[{:keys [marker]}]]- (when (= marker :plugin-a)+ init-steps/do-init-steps! (fn [[{:keys [namespace]}]]+ (when (= namespace (name :plugin-a))(deliver plugin-a-start true)@finish-plugin-a))](is (= :ok (initialize/register-plugin-with-info! (plugin plugin-a-name :plugin-a))))@@ -164,13 +164,14 @@plugin (fn [plugin-name marker]{:metabase-plugin-api-version initialize/plugin-api-version:info {:name plugin-name :version "1.0.0"}- :init [{:step "test" :marker marker}]})]+ :init [{:step "load-namespace" :namespace (name marker)}]})](mt/with-dynamic-fn-redefs [deps/all-dependencies-satisfied? (constantly true)deps/update-unsatisfied-deps! (constantly [])- init-steps/do-init-steps! (fn [[{:keys [marker]}]]- (swap! calls conj marker)- (when (= marker :plugin-a)- (plugins/load-plugin! plugin-b-name)))]+ init-steps/do-init-steps! (fn [[{:keys [namespace]}]]+ (let [marker (keyword namespace)]+ (swap! calls conj marker)+ (when (= marker :plugin-a)+ (plugins/load-plugin! plugin-b-name))))](is (= :ok (initialize/register-plugin-with-info! (plugin plugin-a-name :plugin-a))))(is (= :ok (initialize/register-plugin-with-info! (plugin plugin-b-name :plugin-b))))(is (= :ok (plugins/load-plugin! plugin-a-name))))@@ -184,7 +185,7 @@plugin (fn [plugin-name marker]{:metabase-plugin-api-version initialize/plugin-api-version:info {:name plugin-name :version "1.0.0"}- :init [{:step "test" :marker marker}]})]+ :init [{:step "load-namespace" :namespace (name marker)}]})](mt/with-dynamic-fn-redefs [deps/all-dependencies-satisfied? (constantly true)deps/update-unsatisfied-deps! (constantly [])](is (= :ok (initialize/register-plugin-with-info! (plugin plugin-a-name :plugin-a))))@@ -195,8 +196,8 @@#"while it is already loading"(plugins/load-plugin! plugin-a-name)))))(testing "a cycle through a second plugin"- (mt/with-dynamic-fn-redefs [init-steps/do-init-steps! (fn [[{:keys [marker]}]]- (plugins/load-plugin! (if (= marker :plugin-b)+ (mt/with-dynamic-fn-redefs [init-steps/do-init-steps! (fn [[{:keys [namespace]}]]+ (plugins/load-plugin! (if (= (keyword namespace) :plugin-b)plugin-a-nameplugin-b-name)))](is (thrown-with-msg? clojure.lang.ExceptionInfodiff --git a/test/metabase/pulse/api/pulse_test.clj b/test/metabase/pulse/api/pulse_test.cljindex ba26aa61bd7..301988a16b4 100644--- a/test/metabase/pulse/api/pulse_test.clj+++ b/test/metabase/pulse/api/pulse_test.clj@@ -113,7 +113,10 @@{:cards (str "one or more value must be a map with the following keys ""`(collection_id, description, display, id, include_csv, include_xls, name, dashboard_id, parameter_mappings)`, ""or value must be a map with the keys `id`, `include_csv`, `include_xls`, and `dashboard_card_id`., "- "or value must be a map with the keys `include_csv`, `include_xls`, and `dashboard_card_id`.")}})+ "or value must be a map with the keys `include_csv`, `include_xls`, and `dashboard_card_id`., "+ "or Schema for an instance of a `:model/Card`: every real column of `:report_card` (see `::card.update`) plus `:id`,\n"+ " the `:persisted/*` columns some queries join in from `persisted_info`, and the keys some callers hydrate onto a\n"+ " Card before passing it here.")}})(deftest create-pulse-validation-test(doseq [[input expected-error]@@ -448,7 +451,10 @@{:cards (str "nullable one or more value must be a map with the following keys ""`(collection_id, description, display, id, include_csv, include_xls, name, dashboard_id, parameter_mappings)`, ""or value must be a map with the keys `id`, `include_csv`, `include_xls`, and `dashboard_card_id`., "- "or value must be a map with the keys `include_csv`, `include_xls`, and `dashboard_card_id`.")}})+ "or value must be a map with the keys `include_csv`, `include_xls`, and `dashboard_card_id`., "+ "or Schema for an instance of a `:model/Card`: every real column of `:report_card` (see `::card.update`) plus `:id`,\n"+ " the `:persisted/*` columns some queries join in from `persisted_info`, and the keys some callers hydrate onto a\n"+ " Card before passing it here.")}})(deftest update-pulse-validation-test(testing "PUT /api/pulse/:id"diff --git a/test/metabase/pulse/broken_subscriptions_test.clj b/test/metabase/pulse/broken_subscriptions_test.cljindex 6c187cc6d37..e2c9892db1d 100644--- a/test/metabase/pulse/broken_subscriptions_test.clj+++ b/test/metabase/pulse/broken_subscriptions_test.clj@@ -70,7 +70,7 @@:creator_id (mt/user->id :trashbird):id bad-pulse-id:parameters- [{:name "Source" :slug "source" :id "_SOURCE_PARAM_ID_" :type "string/=" :value ["Twitter" "Facebook"]}]+ [{:name "Source" :slug "source" :id "_SOURCE_PARAM_ID_" :type :string/= :value ["Twitter" "Facebook"]}]:dashboard_id dash-id}{:archived false:name "Bad Slack Pulse"@@ -79,7 +79,7 @@:parameters [{:name "Source":slug "source":id "_SOURCE_PARAM_ID_"- :type "string/="+ :type :string/=:value ["LinkedIn"]}],:dashboard_id dash-id}];; `broken-pulses` doesn't order its results, so sort them for a stable comparison@@ -113,7 +113,7 @@:bad-parameters [{:name "Source":slug "source":id "_SOURCE_PARAM_ID_"- :type "string/="+ :type :string/=:value ["LinkedIn"]}]:dashboard-name "My Awesome Dashboard"}]bad-pulses))))diff --git a/test/metabase/queries/models/card/metadata_test.clj b/test/metabase/queries/models/card/metadata_test.cljindex 1611faae6b0..2f1c553d8cd 100644--- a/test/metabase/queries/models/card/metadata_test.clj+++ b/test/metabase/queries/models/card/metadata_test.clj@@ -4,6 +4,7 @@[metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.queries.models.card.metadata :as card.metadata]+ [metabase.queries.schema :as queries.schema][metabase.test :as mt]))(deftest ^:parallel populate-result-metadata-normalize-output-test@@ -44,7 +45,7 @@:visibility_type :normal:display_name "EDITED DISPLAY":base_type :type/BigInteger}]}- (card.metadata/populate-result-metadata card))))))+ (card.metadata/populate-result-metadata (lib/normalize ::queries.schema/card card)))))))(deftest ^:parallel infer-metadata-no-remaps(testing "infer-metadata should not include remapped columns (#67128)"diff --git a/test/metabase/queries/schema_test.clj b/test/metabase/queries/schema_test.cljindex 6be152229a1..b16bb072009 100644--- a/test/metabase/queries/schema_test.clj+++ b/test/metabase/queries/schema_test.clj@@ -2,6 +2,7 @@(:require[clojure.test :refer :all][malli.error :as me]+ [metabase.lib.core :as lib][metabase.queries.schema :as queries.schema][metabase.util.log.capture :as log.capture][metabase.util.malli.registry :as mr]))@@ -13,11 +14,11 @@(deftest ^:parallel set-invalid-metadata-to-nil-test(is (= {:result_metadata nil}- (queries.schema/normalize-card {:result_metadata [{}]}))))+ (lib/normalize ::queries.schema/card {:result_metadata [{}]}))))(deftest ^:parallel nil-result-metadata-should-not-warn-test(log.capture/with-log-messages-for-level [messages [metabase.queries.schema :warn]](is (= {:result_metadata nil}- (queries.schema/normalize-card {:result_metadata nil})))+ (lib/normalize ::queries.schema/card {:result_metadata nil})))(is (empty? (messages))"nil result_metadata should not produce a warning")))diff --git a/test/metabase/queries_rest/api/card_test.clj b/test/metabase/queries_rest/api/card_test.cljindex ee2e0c8c0b3..905ca288367 100644--- a/test/metabase/queries_rest/api/card_test.clj+++ b/test/metabase/queries_rest/api/card_test.clj@@ -761,11 +761,10 @@:effective_type :type/DateTime:display_name "Timestamp":name "timestamp"- :unit "week"}+ :unit :week}{:base_type :type/Integer:display_name "count"- :name "severity"- :semantic_type :type/Number}]})+ :name "severity"}]})(deftest series-are-compatible-test(testing "area-line-bar charts"diff --git a/test/metabase/query_processor/alternative_date_test.clj b/test/metabase/query_processor/alternative_date_test.cljindex 67589630ff8..aeb7b6a4826 100644--- a/test/metabase/query_processor/alternative_date_test.clj+++ b/test/metabase/query_processor/alternative_date_test.clj@@ -404,13 +404,13 @@:effective-type :type/Text:base-type :type/Text}{:field-name "as_bytes"- :base-type {:natives {:postgres "BYTEA"- :h2 "BYTEA"- :mysql "VARBINARY(100)"- :redshift "VARBYTE"- :presto-jdbc "VARBINARY"- :oracle "BLOB"- :sqlite "BLOB"}}+ :base-type {:natives {"postgres" "BYTEA"+ "h2" "BYTEA"+ "mysql" "VARBINARY(100)"+ "redshift" "VARBYTE"+ "presto-jdbc" "VARBINARY"+ "oracle" "BLOB"+ "sqlite" "BLOB"}}:effective-type :type/DateTime:coercion-strategy :Coercion/YYYYMMDDHHMMSSBytes->Temporal}][["foo" (.getBytes "20190421164300")]@@ -564,13 +564,13 @@:effective-type :type/Text:base-type :type/Text}{:field-name "as_bytes"- :base-type {:natives {:postgres "BYTEA"- :h2 "BYTEA"- :mysql "VARBINARY(100)"- :redshift "VARBYTE"- :presto-jdbc "VARBINARY"- :oracle "BLOB"- :sqlite "BLOB"}}+ :base-type {:natives {"postgres" "BYTEA"+ "h2" "BYTEA"+ "mysql" "VARBINARY(100)"+ "redshift" "VARBYTE"+ "presto-jdbc" "VARBINARY"+ "oracle" "BLOB"+ "sqlite" "BLOB"}}:effective-type :type/DateTime:coercion-strategy :Coercion/ISO8601Bytes->Temporal}][["foo" (.getBytes "2019-04-21 16:43:00")]diff --git a/test/metabase/query_processor/cast_test.clj b/test/metabase/query_processor/cast_test.cljindex f7418ce3933..de60ad53e6c 100644--- a/test/metabase/query_processor/cast_test.clj+++ b/test/metabase/query_processor/cast_test.clj@@ -28,6 +28,11 @@(:import(java.time LocalDate OffsetDateTime)))+(defn- type-info+ "The type keys of result column `col` that [[types/field-is-type?]] reads."+ [col]+ (select-keys col [:base_type :effective_type]))+(set! *warn-on-reflection* true);; integer()@@ -71,7 +76,7 @@result (-> query qp/process-query)cols (mt/cols result)rows (mt/rows result)]- (is (types/field-is-type? :type/Number (last cols)))+ (is (types/field-is-type? :type/Number (type-info (last cols))))(doseq [[uncasted-value casted-value] rows](is (= (biginteger (->integer uncasted-value))(biginteger casted-value))@@ -99,7 +104,7 @@result (-> query qp/process-query)cols (mt/cols result)rows (mt/rows result)]- (is (types/field-is-type? :type/Number (last cols)))+ (is (types/field-is-type? :type/Number (type-info (last cols))))(doseq [[_ uncasted-value casted-value] rows](is (= (biginteger (->integer uncasted-value))(biginteger casted-value))@@ -134,7 +139,7 @@result (-> query qp/process-query)cols (mt/cols result)rows (mt/rows result)]- (is (types/field-is-type? :type/Number (last cols)))+ (is (types/field-is-type? :type/Number (type-info (last cols))))(doseq [[_ uncasted-value casted-value] rows](is (= (biginteger (->integer uncasted-value))(biginteger casted-value))@@ -160,7 +165,7 @@result (-> query qp/process-query)cols (mt/cols result)rows (mt/rows result)]- (is (types/field-is-type? :type/Number (last cols)))+ (is (types/field-is-type? :type/Number (type-info (last cols))))(doseq [[uncasted-value casted-value] rows](is (= (biginteger (->integer uncasted-value))(biginteger casted-value))@@ -195,7 +200,7 @@result (-> query qp/process-query)cols (mt/cols result)rows (mt/rows result)]- (is (types/field-is-type? :type/Number (last cols)))+ (is (types/field-is-type? :type/Number (type-info (last cols))))(doseq [[_ uncasted-value casted-value] rows](is (= (biginteger (->integer uncasted-value))(biginteger casted-value))@@ -223,7 +228,7 @@result (-> query qp/process-query)cols (mt/cols result)rows (mt/rows result)]- (is (types/field-is-type? :type/Number (last cols)))+ (is (types/field-is-type? :type/Number (type-info (last cols))))(doseq [[_ uncasted-value casted-value] rows](is (= (biginteger (->integer uncasted-value))(biginteger casted-value))@@ -244,7 +249,7 @@result (-> query qp/process-query)cols (mt/cols result)rows (mt/rows result)]- (is (types/field-is-type? :type/Number (last cols)))+ (is (types/field-is-type? :type/Number (type-info (last cols))))(doseq [[uncasted-value casted-value] rows](is (= (biginteger (->integer uncasted-value))(biginteger casted-value))@@ -274,7 +279,7 @@result (-> query qp/process-query)cols (mt/cols result)rows (mt/rows result)]- (is (types/field-is-type? :type/Number (last cols)))+ (is (types/field-is-type? :type/Number (type-info (last cols))))(doseq [[_id casted-value] rows](is (= (biginteger value)(biginteger casted-value))@@ -310,7 +315,7 @@result (-> query qp/process-query)cols (mt/cols result)rows (mt/rows result)]- (is (types/field-is-type? :type/Float (last cols)))+ (is (types/field-is-type? :type/Float (type-info (last cols))))(doseq [[uncasted-value casted-value] rows](is (float= (double (Double/parseDouble uncasted-value))(double casted-value))@@ -338,7 +343,7 @@result (-> query qp/process-query)cols (mt/cols result)rows (mt/rows result)]- (is (types/field-is-type? :type/Float (last cols)))+ (is (types/field-is-type? :type/Float (type-info (last cols))))(doseq [[_ uncasted-value casted-value] rows](is (float= (double (Double/parseDouble uncasted-value))(double casted-value))@@ -364,7 +369,7 @@result (-> query qp/process-query)cols (mt/cols result)rows (mt/rows result)]- (is (types/field-is-type? :type/Number (last cols)))+ (is (types/field-is-type? :type/Number (type-info (last cols))))(doseq [[_ uncasted-value casted-value] rows](is (float= (double (Double/parseDouble uncasted-value))(double casted-value))@@ -392,7 +397,7 @@result (-> query qp/process-query)cols (mt/cols result)rows (mt/rows result)]- (is (types/field-is-type? :type/Number (last cols)))+ (is (types/field-is-type? :type/Number (type-info (last cols))))(doseq [[uncasted-value casted-value] rows](is (float= (double (Double/parseDouble uncasted-value))(double casted-value))@@ -429,7 +434,7 @@result (-> query qp/process-query)cols (mt/cols result)rows (mt/rows result)]- (is (types/field-is-type? :type/Number (last cols)))+ (is (types/field-is-type? :type/Number (type-info (last cols))))(doseq [[_id uncasted-value casted-value] rows](is (float= (double (Double/parseDouble uncasted-value))(double casted-value))@@ -456,7 +461,7 @@result (-> query qp/process-query)cols (mt/cols result)rows (mt/rows result)]- (is (types/field-is-type? :type/Number (last cols)))+ (is (types/field-is-type? :type/Number (type-info (last cols))))(doseq [[_ uncasted-value casted-value] rows](is (float= (double (Double/parseDouble uncasted-value))(double casted-value))@@ -476,7 +481,7 @@result (-> query qp/process-query)cols (mt/cols result)rows (mt/rows result)]- (is (types/field-is-type? :type/Number (last cols)))+ (is (types/field-is-type? :type/Number (type-info (last cols))))(doseq [[uncasted-value casted-value] rows](is (float= (double (Double/parseDouble uncasted-value))(double casted-value))@@ -502,7 +507,7 @@result (-> query qp/process-query)cols (mt/cols result)rows (mt/rows result)]- (is (types/field-is-type? :type/Number (last cols)))+ (is (types/field-is-type? :type/Number (type-info (last cols))))(doseq [[_id casted-value] rows](is (float= (double value)(double casted-value))@@ -554,7 +559,7 @@cols (mt/cols result)rows (mt/rows result)](is (types/field-is-type? (date-type-expected driver/*driver*)- (last cols)))+ (type-info (last cols))))(doseq [[_ uncasted-value casted-value] rows](let [cd (parse-date casted-value)ud (parse-date uncasted-value)]@@ -597,7 +602,7 @@result (-> query qp/process-query)cols (mt/cols result)rows (mt/rows result)]- (is (types/field-is-type? :type/Text (last cols)))+ (is (types/field-is-type? :type/Text (type-info (last cols))))(doseq [[_id casted-value] rows](is (string? casted-value))(is (compare casted-value expected) (str "Not equal for " msg)))))))))@@ -616,7 +621,7 @@result (-> query qp/process-query)cols (mt/cols result)rows (mt/rows result)]- (is (types/field-is-type? :type/Text (last cols)))+ (is (types/field-is-type? :type/Text (type-info (last cols))))(doseq [[_id casted-value] rows](is (string? casted-value))(is (compare casted-value expected) (str "Not equal for " msg)))))))))@@ -639,7 +644,7 @@result (-> query qp/process-query)cols (mt/cols result)rows (mt/rows result)]- (is (types/field-is-type? :type/Text (last cols)))+ (is (types/field-is-type? :type/Text (type-info (last cols))))(doseq [[_field casted-value] rows](is (string? casted-value)))))))))@@ -664,7 +669,7 @@result (-> query qp/process-query)cols (mt/cols result)rows (mt/rows result)]- (is (types/field-is-type? :type/Text (last cols)))+ (is (types/field-is-type? :type/Text (type-info (last cols))))(doseq [[_id _uncasted casted-value] rows](is (string? casted-value)))))))))@@ -698,7 +703,7 @@result (-> query qp/process-query)cols (mt/cols result)rows (mt/rows result)]- (is (types/field-is-type? :type/Text (last cols)))+ (is (types/field-is-type? :type/Text (type-info (last cols))))(doseq [[_id _expression casted-value] rows](is (string? casted-value))(is (= expected casted-value)))))))))@@ -725,7 +730,7 @@result (-> query qp/process-query)cols (mt/cols result)rows (mt/rows result)]- (is (types/field-is-type? :type/Text (last cols)))+ (is (types/field-is-type? :type/Text (type-info (last cols))))(doseq [[_uncasted-value casted-value] rows](is (string? casted-value)))))))))@@ -755,7 +760,7 @@result (-> query qp/process-query)cols (mt/cols result)rows (mt/rows result)]- (is (types/field-is-type? :type/Text (last cols)))+ (is (types/field-is-type? :type/Text (type-info (last cols))))(doseq [[_id _uncasted casted-value] rows](is (string? casted-value)))))))))@@ -780,7 +785,7 @@result (-> query qp/process-query)cols (mt/cols result)rows (mt/rows result)]- (is (types/field-is-type? :type/Text (last cols)))+ (is (types/field-is-type? :type/Text (type-info (last cols))))(doseq [[_id _uncasted casted-value] rows](is (string? casted-value))))))))@@ -801,17 +806,18 @@result (-> query qp/process-query)cols (mt/cols result)rows (mt/rows result)]- (is (types/field-is-type? :type/Text (last cols)))+ (is (types/field-is-type? :type/Text (type-info (last cols))))(doseq [[_uncasted-value casted-value] rows](is (string? casted-value)))))))));; datetime()(defn- datetime-type? [col]- (some #(types/field-is-type? % col) [:type/DateTime ;; some databases return datetimes for date (e.g., Oracle)- :type/Text ;; sqlite uses text :(- :type/* ;; Mongo- ]))+ (let [col (type-info col)]+ (some #(types/field-is-type? % col) [:type/DateTime ;; some databases return datetimes for date (e.g., Oracle)+ :type/Text ;; sqlite uses text :(+ :type/* ;; Mongo+ ])))(deftest ^:parallel datetime-cast(mt/test-drivers (mt/normal-drivers-with-feature :expressions/datetime)@@ -859,13 +865,13 @@:effective-type :type/Text:base-type :type/Text}{:field-name "as_bytes"- :base-type {:natives {:postgres "BYTEA"- :h2 "BYTEA"- :mysql "VARBINARY(100)"- :redshift "VARBYTE"- :presto-jdbc "VARBINARY"- :oracle "BLOB"- :sqlite "BLOB"}}}]+ :base-type {:natives {"postgres" "BYTEA"+ "h2" "BYTEA"+ "mysql" "VARBINARY(100)"+ "redshift" "VARBYTE"+ "presto-jdbc" "VARBINARY"+ "oracle" "BLOB"+ "sqlite" "BLOB"}}}][["foo" (.getBytes "20190421164300")]["bar" (.getBytes "20200421164300")]["baz" (.getBytes "20210421164300")]]]])@@ -875,13 +881,13 @@:effective-type :type/Text:base-type :type/Text}{:field-name "as_bytes"- :base-type {:natives {:postgres "BYTEA"- :h2 "BYTEA"- :mysql "VARBINARY(100)"- :redshift "VARBYTE"- :presto-jdbc "VARBINARY"- :oracle "BLOB"- :sqlite "BLOB"}}}]+ :base-type {:natives {"postgres" "BYTEA"+ "h2" "BYTEA"+ "mysql" "VARBINARY(100)"+ "redshift" "VARBYTE"+ "presto-jdbc" "VARBINARY"+ "oracle" "BLOB"+ "sqlite" "BLOB"}}}][["foo" (.getBytes "2019-04-21 16:43:00")]["bar" (.getBytes "2020-04-21T16:43:00")]["baz" (.getBytes "2021-04-21 16:43:00")]]]])@@ -1075,7 +1081,7 @@cols (mt/cols result)rows (mt/rows result)](is (types/field-is-type? (date-type-expected driver/*driver*)- (last cols)))+ (type-info (last cols))))(doseq [[_id today] rows](is (= (parse-date today)(LocalDate/now))))))))diff --git a/test/metabase/query_processor/coercion_test.clj b/test/metabase/query_processor/coercion_test.cljindex c5aa55e32cd..03cd3344926 100644--- a/test/metabase/query_processor/coercion_test.clj+++ b/test/metabase/query_processor/coercion_test.clj@@ -79,9 +79,10 @@(defn- date-type? [col];; legacy usage -- do not use going forward#_{:clj-kondo/ignore [:deprecated-var]}- (some #(types/field-is-type? % col) [:type/DateTime ;; some databases return datetimes for date (e.g., Oracle)- :type/Text ;; sqlite uses text :(- :type/Date]))+ (let [col (select-keys col [:base_type :effective_type])]+ (some #(types/field-is-type? % col) [:type/DateTime ;; some databases return datetimes for date (e.g., Oracle)+ :type/Text ;; sqlite uses text :(+ :type/Date])))(defn- parse-date [s](trydiff --git a/test/metabase/query_processor/json_field_alias_test.clj b/test/metabase/query_processor/json_field_alias_test.cljindex 219b5705123..2f97d2ab383 100644--- a/test/metabase/query_processor/json_field_alias_test.clj+++ b/test/metabase/query_processor/json_field_alias_test.clj@@ -22,7 +22,7 @@;; force the driver namespace to load so its hierarchy derivations and `database-supports?`;; defmethods are registered.:let [_ (driver/the-driver driver)- stub-db {:lib/type :metadata/database, :engine driver, :id -1, :details {}}]+ stub-db {:lib/type :metadata/database, :engine driver, :id 1, :details {}}]:when (and (isa? driver/hierarchy driver :sql)(or (driver/database-supports? driver :nested-field-columns stub-db)(driver/database-supports? driver :nested-fields stub-db)))]diff --git a/test/metabase/query_processor/middleware/add_implicit_joins_test.clj b/test/metabase/query_processor/middleware/add_implicit_joins_test.cljindex d44a5410d0a..68af16e22d2 100644--- a/test/metabase/query_processor/middleware/add_implicit_joins_test.clj+++ b/test/metabase/query_processor/middleware/add_implicit_joins_test.clj@@ -7,6 +7,7 @@[metabase.lib.metadata :as lib.metadata][metabase.lib.options :as lib.options][metabase.lib.schema.join :as lib.schema.join]+ [metabase.lib.schema.metadata :as lib.schema.metadata][metabase.lib.test-metadata :as meta][metabase.lib.test-util :as lib.tu][metabase.lib.test-util.macros :as lib.tu.macros]@@ -79,11 +80,11 @@:query)))))(mu/defn- add-implicit-joins :- ::qp.schema/any-query- ([query]+ ([query :- ::qp.schema/any-query](add-implicit-joins meta/metadata-provider query))- ([metadata-provider- query :- ::qp.schema/any-query]+ ([metadata-provider :- ::lib.schema.metadata/metadata-providerable+ query :- ::qp.schema/any-query](if (:lib/type query)(qp.add-implicit-joins/add-implicit-joins query)(-> (lib/query metadata-provider query)diff --git a/test/metabase/query_processor/middleware/add_remaps_test.clj b/test/metabase/query_processor/middleware/add_remaps_test.cljindex fc0b3dd5110..297d55a4158 100644--- a/test/metabase/query_processor/middleware/add_remaps_test.clj+++ b/test/metabase/query_processor/middleware/add_remaps_test.clj@@ -200,18 +200,21 @@;;; ---------------------------------------- remap-results (post-processing) -----------------------------------------(defn- remap-results [query metadata rows]- (let [rff (qp.add-remaps/remap-results query qp.reducible/default-rff)+ (let [query (merge (lib/query (qp.store/metadata-provider) (lib.metadata/table (qp.store/metadata-provider) (meta/id :venues)))+ query)+ rff (qp.add-remaps/remap-results query qp.reducible/default-rff)rf (rff metadata)](transduce identity rf rows)))(defn venues-column-metadata []- (lib.metadata/bulk-metadata-or-throw- (qp.store/metadata-provider)- :metadata/column- [(meta/id :venues :id)- (meta/id :venues :name)- (meta/id :venues :category-id)- (meta/id :venues :price)]))+ (mapv lib/lib-metadata-column->legacy-metadata-column+ (lib.metadata/bulk-metadata-or-throw+ (qp.store/metadata-provider)+ :metadata/column+ [(meta/id :venues :id)+ (meta/id :venues :name)+ (meta/id :venues :category-id)+ (meta/id :venues :price)])))(deftest ^:parallel remap-human-readable-values-test(testing "remapping columns with `human_readable_values`"@@ -392,13 +395,16 @@metadata (lib.tu.macros/$ids venues{:cols [{:name "NAME":id %name+ :base_type :type/Text:display_name "Name"}{:name "CATEGORY_ID":id %category-id+ :base_type :type/Integer:display_name "Category ID":options {::qp.add-remaps/original-field-dimension-id dimension-id}}{:name "NAME_2":id %categories.name+ :base_type :type/Text:display_name "Category → Name":fk_field_id %category-id:options {::qp.add-remaps/new-field-dimension-id dimension-id}}]})]@@ -473,22 +479,27 @@(let [metadata (lib.tu.macros/$ids venues{:cols [{:name "CATEGORY_ID":id %category-id+ :base_type :type/Integer:display_name "Category ID":options {::qp.add-remaps/original-field-dimension-id category-id-dimension-id}}{:name "ID":id %id+ :base_type :type/BigInteger:display_name "ID":options {::qp.add-remaps/original-field-dimension-id id-dimension-id}}{:name "NAME":id %name+ :base_type :type/Text:display_name "Name"}{:name "NAME_2":id %categories.name+ :base_type :type/Text:display_name "Categories → Name":fk_field_id %category-id:options {::qp.add-remaps/new-field-dimension-id category-id-dimension-id}}{:name "NAME_3":id %categories.name+ :base_type :type/Text:display_name "Categories → Name":fk_field_id %id:options {::qp.add-remaps/new-field-dimension-id id-dimension-id}}]})]diff --git a/test/metabase/query_processor/middleware/add_rows_truncated_test.clj b/test/metabase/query_processor/middleware/add_rows_truncated_test.cljindex 4e2e98100f8..19330e0729c 100644--- a/test/metabase/query_processor/middleware/add_rows_truncated_test.clj+++ b/test/metabase/query_processor/middleware/add_rows_truncated_test.clj@@ -13,7 +13,7 @@(defn- add-rows-truncated [query rows](let [rff (add-rows-truncated/add-rows-truncated (assoc query :lib/type :mbql/query) qp.reducible/default-rff)- rf (rff nil)]+ rf (rff {})](transduce identity rf rows)))(deftest ^:parallel add-rows-truncated-testdiff --git a/test/metabase/query_processor/middleware/annotate/legacy_helper_fns_test.clj b/test/metabase/query_processor/middleware/annotate/legacy_helper_fns_test.cljindex 876e2b17067..e7e7592a428 100644--- a/test/metabase/query_processor/middleware/annotate/legacy_helper_fns_test.clj+++ b/test/metabase/query_processor/middleware/annotate/legacy_helper_fns_test.clj@@ -2,6 +2,7 @@(:require[clojure.test :refer :all][clojure.walk :as walk]+ [metabase.lib.core :as lib][metabase.lib.test-metadata :as meta][metabase.query-processor.middleware.annotate.legacy-helper-fns :as annotate.legacy-helper-fns];; binds mock metadata providers via the ambient store, which the code under test reads@@ -14,7 +15,8 @@(fn [form](cond-> form(keyword? form) u/qualified-name))- (meta/field-metadata :venues :name))]+ (lib/lib-metadata-column->legacy-metadata-column+ (meta/field-metadata :venues :name)))]:aggregation [[:count]]}](qp.store/with-metadata-provider meta/metadata-provider(is (= "count"diff --git a/test/metabase/query_processor/middleware/annotate_test.clj b/test/metabase/query_processor/middleware/annotate_test.cljindex e1945b6d712..1d8052d8e00 100644--- a/test/metabase/query_processor/middleware/annotate_test.clj+++ b/test/metabase/query_processor/middleware/annotate_test.clj@@ -7,6 +7,7 @@[metabase.config.core :as config][metabase.driver :as driver][metabase.lib.core :as lib]+ [metabase.lib.schema :as lib.schema][metabase.lib.test-metadata :as meta][metabase.lib.test-util :as lib.tu][metabase.lib.test-util.macros :as lib.tu.macros]@@ -17,7 +18,8 @@[metabase.test :as mt][metabase.test.fixtures :as fixtures][metabase.util :as u]- [metabase.util.malli :as mu]))+ [metabase.util.malli :as mu]+ [metabase.util.malli.schema :as ms]))(set! *warn-on-reflection* true)@@ -29,12 +31,13 @@{:cols [{:name "_id"} {:name "longitude"} {:name "category_id"} {:name "price"} {:name "name"} {:name "latitude"}]})))(mu/defn- add-column-info- ([query metadata]+ ([query :- ::lib.schema/query+ metadata :- ::annotate/metadata](add-column-info query metadata []))- ([query :- :map+ ([query :- ::lib.schema/querymetadata :- ::annotate/metadata- rows :- [:maybe [:sequential [:sequential :any]]]]+ rows :- [:maybe [:sequential [:sequential ms/FieldValue]]]](letfn [(rff [metadata](fn rf([]@@ -53,14 +56,14 @@(testing "native column info"(testing "should still infer types even if the initial value(s) are `nil` (#4256, #6924)"(is (= [:type/Integer]- (transduce identity (#'annotate/base-type-inferer {:cols [{}]})+ (transduce identity (#'annotate/base-type-inferer {:cols [{:name "a"}]})(concat (repeat 1000 [nil]) [[1] [2]])))))))(deftest ^:parallel native-column-info-test-2(testing "native column info"(testing "should use default `base_type` of `type/*` if there are no non-nil values in the sample"(is (= [:type/*]- (transduce identity (#'annotate/base-type-inferer {:cols [{}]})+ (transduce identity (#'annotate/base-type-inferer {:cols [{:name "a"}]})[[nil]]))))))(deftest ^:parallel native-column-info-test-3@@ -68,7 +71,7 @@(testing "should attempt to infer better base type if driver returns :type/* (#12150)";; `merged-column-info` handles merging info returned by driver & inferred by annotate(is (= [:type/Integer]- (transduce identity (#'annotate/base-type-inferer {:cols [{:base_type :type/*}]})+ (transduce identity (#'annotate/base-type-inferer {:cols [{:name "a", :base_type :type/*}]})[[1] [2] [nil] [3]]))))))(defn- column-info [query {:keys [rows], :as metadata}]@@ -199,7 +202,7 @@{:name "count"}]]:limit 1})))](is (= ["Sum of Sum of Price" "Count"]- (->> (add-column-info query {:cols [{} {}]})+ (->> (add-column-info query {:cols [{:name "sum"} {:name "count"}]}):cols(map :display_name))))))))@@ -357,7 +360,7 @@(testing "In case of a lib vs. driver column count mismatch, don't loop infinitely (#66955)"(let [query (lib/query meta/metadata-provider(meta/table-metadata :orders))- all-cols (mapv #(select-keys % [:name :base-type]) (lib/returned-columns query))+ all-cols (mapv (fn [col] {:name (:name col), :base_type (:base-type col)}) (lib/returned-columns query))missing-one (butlast all-cols)](is (= 9 (count (annotate/expected-cols query all-cols))))(testing "in dev and test modes, we throw an error when the counts differ"diff --git a/test/metabase/query_processor/middleware/auto_bucket_datetimes_test.clj b/test/metabase/query_processor/middleware/auto_bucket_datetimes_test.cljindex d92053c7484..d2ecb4c82a1 100644--- a/test/metabase/query_processor/middleware/auto_bucket_datetimes_test.clj+++ b/test/metabase/query_processor/middleware/auto_bucket_datetimes_test.clj@@ -1,9 +1,11 @@(ns metabase.query-processor.middleware.auto-bucket-datetimes-test(:require[clojure.test :refer :all]+ [metabase.legacy-mbql.schema :as mbql.s][metabase.lib.convert :as lib.convert][metabase.lib.core :as lib][metabase.lib.normalize :as lib.normalize]+ [metabase.lib.schema :as lib.schema][metabase.lib.test-metadata :as meta][metabase.lib.test-util :as lib.tu][metabase.lib.test-util.macros :as lib.tu.macros]@@ -20,7 +22,7 @@[:stages 0](lib.normalize/normalize :mbql.clause/field [:field {:temporal-unit :month} (meta/id :checkins :date)]))))))-(mu/defn- auto-bucket [query :- :map]+(mu/defn- auto-bucket [query :- [:or ::lib.schema/query ::mbql.s/Query]](if (= (:lib/type query) :mbql/query)(qp.auto-bucket-datetimes/auto-bucket-datetimes query)(let [metadata-provider (if (qp.store/initialized?)diff --git a/test/metabase/query_processor/middleware/cache_test.clj b/test/metabase/query_processor/middleware/cache_test.cljindex 6b350bbd305..bc7a06b0b2a 100644--- a/test/metabase/query_processor/middleware/cache_test.clj+++ b/test/metabase/query_processor/middleware/cache_test.clj@@ -1015,18 +1015,18 @@;; An rff that injects a sentinel `:fresh` into metadata, the same shape;; `update-viz-settings` uses to inject fresh viz-settings on cache hit.fresh-injecting-rff (fn [metadata]- (qp.reducible/default-rff (assoc metadata :fresh "fresh-value")))+ (qp.reducible/default-rff (assoc metadata :viz-settings {:fresh "fresh-value"})))rf ((cached-results-rff fresh-injecting-rff (byte-array 1))- {:last-ran (t/zoned-date-time) :cache-version "v1" :cols [{:name "x"}]})+ {:last-ran (t/zoned-date-time) :cache-version 1 :cols [{:name "x"}]});; Simulate a cached replay: two actual row vectors, then the final cached;; result map (which used to stomp acc).acc (reduce rf (rf)[[1] [2]- {:data {:cols [{:name "x"}] :stale "stale-value"}}])+ {:data {:cols [{:name "x"}] :results_timezone "stale-value"}}])result (rf acc)]- (is (= "fresh-value" (get-in result [:data :fresh]))+ (is (= "fresh-value" (get-in result [:data :viz-settings :fresh]))"Fresh value injected by the rff chain must survive the cached-final-metadata replay")- (is (= "stale-value" (get-in result [:data :stale]))+ (is (= "stale-value" (get-in result [:data :results_timezone]))"Stale-only keys from @final-metadata are still deep-merged in")(is (= [[1] [2]] (get-in result [:data :rows]))"Replayed cached rows are preserved"))))diff --git a/test/metabase/query_processor/middleware/catch_exceptions_test.clj b/test/metabase/query_processor/middleware/catch_exceptions_test.cljindex b6627011688..2595b34ecd5 100644--- a/test/metabase/query_processor/middleware/catch_exceptions_test.clj+++ b/test/metabase/query_processor/middleware/catch_exceptions_test.clj@@ -171,16 +171,16 @@(testing "Should include info from QueryExecution if added to the thrown/raised Exception"(binding [qp.pipeline/*run* (fn [_query _rff](throw (ex-info "Something went wrong."- {:query-execution {:a 100- :b 200+ {:query-execution {:context :ad-hoc+ :lens_id "lens":card_id 300;; these keys should all get removed:result_rows 400- :hash 500+ :hash (byte-array [5]):executor_id 500:dashboard_id 700:pulse_id 800- :native 900}}+ :native true}}(Exception. "Something went wrong"))))](is (=? {:status :failed:class (partial = java.lang.Exception)@@ -190,8 +190,8 @@:json_query {}:row_count 0:data {:cols []}- :a 100- :b 200}+ :context :ad-hoc+ :lens_id "lens"}(catch-exceptions (fn run []) {})))))))(deftest ^:parallel catch-exceptions-test-2diff --git a/test/metabase/query_processor/middleware/cumulative_aggregations_test.clj b/test/metabase/query_processor/middleware/cumulative_aggregations_test.cljindex 1c4a26acac7..59022d6d1f8 100644--- a/test/metabase/query_processor/middleware/cumulative_aggregations_test.clj+++ b/test/metabase/query_processor/middleware/cumulative_aggregations_test.clj@@ -146,8 +146,9 @@[#t "2016-04-04" "San Francisco" 40] ; [SF] 40 + 40 => 80[#t "2016-04-06" "Long Beach" 8]] ; [LB] 6 + 8 => 14rff (qp.cumulative-aggregations/sum-cumulative-aggregation-columns- {:stages [{:breakout [:a :b]- ::qp.cumulative-aggregations/replaced-indexes #{2}}]}+ {:lib/type :mbql/query+ :stages [{:breakout [:a :b]+ ::qp.cumulative-aggregations/replaced-indexes #{2}}]}(fn [_metadata]conj))rf (rff nil)]@@ -171,8 +172,9 @@[#t "2016-04-06" "Long Beach" "LB" 8] ; [LB] 6 + 8 => 14[#t "2016-04-06" "Long Beach" "LBC" 8]] ; [LBC] 4 + 8 => 12rff (qp.cumulative-aggregations/sum-cumulative-aggregation-columns- {:stages [{:breakout [:a :b :c]- ::qp.cumulative-aggregations/replaced-indexes #{3}}]}+ {:lib/type :mbql/query+ :stages [{:breakout [:a :b :c]+ ::qp.cumulative-aggregations/replaced-indexes #{3}}]}(fn [_metadata]conj))rf (rff nil)]diff --git a/test/metabase/query_processor/middleware/parameters_test.clj b/test/metabase/query_processor/middleware/parameters_test.cljindex 5d7581640b3..3548a04bacf 100644--- a/test/metabase/query_processor/middleware/parameters_test.clj+++ b/test/metabase/query_processor/middleware/parameters_test.clj@@ -310,12 +310,10 @@meta/metadata-provider{:native-query-snippets [{:id 1:content "name, price"- :creator-id (mt/user->id :rasta):description "Fields to SELECT":name "Venue fields"}{:id 2:content "price > 2"- :creator-id (mt/user->id :rasta):description "Meant for use in WHERE clause":name "Filter: expensive venues"}]:cards [{:id 1diff --git a/test/metabase/query_processor/middleware/visualization_settings_test.clj b/test/metabase/query_processor/middleware/visualization_settings_test.cljindex 98d1cbbf19d..c3991ac6611 100644--- a/test/metabase/query_processor/middleware/visualization_settings_test.clj+++ b/test/metabase/query_processor/middleware/visualization_settings_test.clj@@ -121,7 +121,7 @@(let [mp (lib.tu/mock-metadata-providercard-viz-settings-test-mock-metadata-provider{:cards [{:id 1- :datset-query {:database 1, :type :native, :native {:query "X"}}+ :dataset-query {:database 1, :type :native, :native {:query "X"}}:database-id 1:visualization-settings (db-viz-settings 1 2)}]})query (lib/query mp (test-query [1 2 3] 1 nil))diff --git a/test/metabase/query_processor/nested_queries_test.clj b/test/metabase/query_processor/nested_queries_test.cljindex 5be5b4de5e0..1bb830d4e61 100644--- a/test/metabase/query_processor/nested_queries_test.clj+++ b/test/metabase/query_processor/nested_queries_test.clj@@ -752,7 +752,7 @@(qp.store/with-metadata-provider (-> meta/metadata-provider(lib.tu/metadata-provider-with-cards-for-queries [{}])(lib.tu/merged-mock-metadata-provider {:cards [{:id 1, :collection-id 1000}]}))- (is (= {:paths #{(perms/collection-read-path (t2/instance :model/Collection {:id 1000}))}}+ (is (= {:paths #{(perms/collection-read-path 1000)}}(query-perms/required-perms-for-query (query-with-source-card 1lib.schema.id/saved-questions-virtual-database-id:aggregation [:count]))))))))diff --git a/test/metabase/query_processor/parameters/operators_test.clj b/test/metabase/query_processor/parameters/operators_test.cljindex 605078818e6..84977c9e557 100644--- a/test/metabase/query_processor/parameters/operators_test.clj+++ b/test/metabase/query_processor/parameters/operators_test.clj@@ -116,7 +116,7 @@(testing "between normalization"(are [values op] (=? [op {} [:field {:source-field 5} 26] 1](params.ops/to-clause {:type :number/between- :target [:dimension [:field {:source-field 5} 26]]+ :target [:dimension [:field 26 {:source-field 5}]]:value values}))[1] :>=[1 nil] :>=diff --git a/test/metabase/query_processor/parameters/values_test.clj b/test/metabase/query_processor/parameters/values_test.cljindex c1458b1a21a..876d4a5e12b 100644--- a/test/metabase/query_processor/parameters/values_test.clj+++ b/test/metabase/query_processor/parameters/values_test.clj@@ -362,7 +362,7 @@nil))))))(mu/defn- query->params-map- ([inner-query]+ ([inner-query :- ::mbql.s/SourceQuery](query->params-map meta/metadata-provider inner-query))([metadata-provider :- ::lib.schema.metadata/metadata-providerinner-query :- ::mbql.s/SourceQuery]@@ -388,7 +388,7 @@mp (lib.tu/metadata-provider-with-cards-for-queriesmeta/metadata-provider[{:database (meta/id)- :type "native"+ :type :native:native {:query test-query}}])](is (=? {:card-id 1, :query test-query, :parameters nil}(value-for-tag@@ -404,8 +404,7 @@(testing "Card query template tag generates native query for MBQL query"(driver/with-driver :h2(let [mbql-query (lib.tu.macros/mbql-query venues- {:database (meta/id)- :filter [:< [:field $price nil] 3]})+ {:filter [:< $price 3]})expected-sql (str "SELECT ""\"PUBLIC\".\"VENUES\".\"ID\" AS \"ID\", ""\"PUBLIC\".\"VENUES\".\"NAME\" AS \"NAME\", "@@ -508,7 +507,7 @@:template-tags {"#1" {:id "#1":name "#1":display-name "#1"- :type "card"+ :type :card:card-id 1}}}:database (meta/id)}])tag {:name "card-template-tag-test"diff --git a/test/metabase/query_processor/pivot/middleware_test.clj b/test/metabase/query_processor/pivot/middleware_test.cljindex cf02ce2cd80..0e98ed3d2fb 100644--- a/test/metabase/query_processor/pivot/middleware_test.clj+++ b/test/metabase/query_processor/pivot/middleware_test.clj@@ -1,13 +1,16 @@(ns metabase.query-processor.pivot.middleware-test(:require[clojure.test :refer :all]+ [metabase.lib.core :as lib]+ [metabase.lib.test-metadata :as meta][metabase.query-processor.pivot.middleware :as qp.pivot.middleware]))(deftest ^:parallel full-breakout-combination-test(letfn [(combo [breakout-combination remaps](#'qp.pivot.middleware/full-breakout-combination- {:qp.pivot/remapped-breakout-combination breakout-combination- :qp.pivot/remapped-indexes remaps}))]+ (assoc (lib/query meta/metadata-provider (meta/table-metadata :venues))+ :qp.pivot/remapped-breakout-combination breakout-combination+ :qp.pivot/remapped-indexes remaps)))](is (= [](combo [] {1 0, 4 3})))(is (= [0 1]diff --git a/test/metabase/query_processor/postprocess_test.clj b/test/metabase/query_processor/postprocess_test.cljindex cfc399c27eb..e4dd1e8a104 100644--- a/test/metabase/query_processor/postprocess_test.clj+++ b/test/metabase/query_processor/postprocess_test.clj@@ -10,7 +10,7 @@(set! *warn-on-reflection* true)(deftest ^:parallel row-type-agnostic-test- (let [metadata {:cols [{:base_type :type/Integer}]}+ (let [metadata {:cols [{:name "ID", :base_type :type/Integer}]}api-qp-middleware-options (delay (-> (mt/user-http-request :rasta :post 202 "dataset" (mt/mbql-query users {:limit 1})):json_query:middleware))]diff --git a/test/metabase/query_processor/streaming/csv_test.clj b/test/metabase/query_processor/streaming/csv_test.cljindex faecc713070..93231817691 100644--- a/test/metabase/query_processor/streaming/csv_test.clj+++ b/test/metabase/query_processor/streaming/csv_test.clj@@ -134,9 +134,9 @@(defn- csv-export"Given a seq of result rows, write it as a CSV, then read the CSV and return the resulting data."[rows]- (rest (csv-export-with-cols [{:base_type :type/*}- {:base_type :type/*}- {:base_type :type/*}]+ (rest (csv-export-with-cols [{:name "A", :display_name "A", :base_type :type/*}+ {:name "B", :display_name "B", :base_type :type/*}+ {:name "C", :display_name "C", :base_type :type/*}]rows)))(deftest csv-export-includes-utf8-bom-test@@ -146,7 +146,7 @@(with-open [bos (ByteArrayOutputStream.)os (BufferedOutputStream. bos)](let [results-writer (qp.si/streaming-results-writer :csv os)]- (qp.si/begin! results-writer {:data {:ordered-cols [{:base_type :type/*}]}} {})+ (qp.si/begin! results-writer {:data {:ordered-cols [{:name "A", :display_name "A", :base_type :type/*}]}} {})(qp.si/write-row! results-writer ["£37.65"] 0 [] {})(qp.si/finish! results-writer {:row_count 1}))(is (= [(unchecked-byte 0xEF) (unchecked-byte 0xBB) (unchecked-byte 0xBF)]@@ -159,7 +159,7 @@(with-open [bos (ByteArrayOutputStream.)os (BufferedOutputStream. bos)](let [results-writer (qp.si/streaming-results-writer :csv os)]- (qp.si/begin! results-writer {:data {:ordered-cols [{:base_type :type/*}] :csv-include-bom? false}} {})+ (qp.si/begin! results-writer {:data {:ordered-cols [{:name "A", :display_name "A", :base_type :type/*}] :csv-include-bom? false}} {})(qp.si/write-row! results-writer ["£37.65"] 0 [] {})(qp.si/finish! results-writer {:row_count 1}))(let [bytes (seq (.toByteArray bos))]diff --git a/test/metabase/query_processor/test_util.clj b/test/metabase/query_processor/test_util.cljindex 048a4d82bf7..cff19ccb1c7 100644--- a/test/metabase/query_processor/test_util.clj+++ b/test/metabase/query_processor/test_util.clj@@ -504,13 +504,13 @@(mu/defn metadata-provider-with-cards-for-queries :- ::lib.schema.metadata/metadata-provider"Create an Lib metadata provider (by default, based on the app DB metadata provider) that adds a Card for each queryin `queries`. Cards do not include result metadata. Cards have IDs starting at `1` and increasing sequentially."- ([queries]+ ([queries :- [:sequential {:min 1} ::lib.schema.metadata/card.query]](metadata-provider-with-cards-for-queries(lib-be/application-database-metadata-provider (data/id))queries))([parent-metadata-provider :- ::lib.schema.metadata/metadata-provider- queries :- [:sequential {:min 1} :map]]+ queries :- [:sequential {:min 1} ::lib.schema.metadata/card.query]](lib.tu/metadata-provider-with-cards-for-queries parent-metadata-provider queries)))(mu/defn metadata-provider-with-cards-with-transformed-metadata-for-queries :- ::lib.schema.metadata/metadata-provider@@ -519,16 +519,16 @@provider is built up progressively, meaning metadata for previous Cards is available when calculating metadata forsubsequent Cards.`transforms` can be a map of `card-id` to a function that accepts `metadata-provider` and `result-metadata`"- ([queries :- [:sequential {:min 1} :map]- transforms]+ ([queries :- [:sequential {:min 1} ::lib.schema.metadata/card.query]+ transforms :- [:maybe [:map-of pos-int? fn?]]](metadata-provider-with-cards-with-transformed-metadata-for-queries(lib-be/application-database-metadata-provider (data/id))queriestransforms))([parent-metadata-provider :- ::lib.schema.metadata/metadata-provider- queries :- [:sequential {:min 1} :map]- transforms :- [:maybe :map]]+ queries :- [:sequential {:min 1} ::lib.schema.metadata/card.query]+ transforms :- [:maybe [:map-of pos-int? fn?]]](transduce(map-indexed (fn [i {database-id :database, :as query}]{:id (inc i)@@ -556,13 +556,13 @@of [[metabase.query-processor.preprocess/query->expected-cols]] as `:result-metadata` for each Card. The metadataprovider is built up progressively, meaning metadata for previous Cards is available when calculating metadata forsubsequent Cards."- ([queries]+ ([queries :- [:sequential {:min 1} ::lib.schema.metadata/card.query]](metadata-provider-with-cards-with-metadata-for-queries(lib-be/application-database-metadata-provider (data/id))queries))([parent-metadata-provider :- ::lib.schema.metadata/metadata-provider- queries :- [:sequential {:min 1} :map]]+ queries :- [:sequential {:min 1} ::lib.schema.metadata/card.query]](metadata-provider-with-cards-with-transformed-metadata-for-queries parent-metadata-provider queries nil)))(deftest ^:parallel metadata-provider-with-cards-with-metadata-for-queries-testdiff --git a/test/metabase/query_processor/timezones_test.clj b/test/metabase/query_processor/timezones_test.cljindex 05555f085cd..a387022a015 100644--- a/test/metabase/query_processor/timezones_test.clj+++ b/test/metabase/query_processor/timezones_test.clj@@ -113,7 +113,8 @@(defn- field-identifier [table-key field-key](sql.qp/->honeysql driver/*driver*[:field- {:metabase.query-processor.util.add-alias-info/source-table (mt/id table-key)}+ {:lib/uuid (str (random-uuid))+ :metabase.query-processor.util.add-alias-info/source-table (mt/id table-key)}(mt/id table-key field-key)]))(defn- honeysql->sql [honeysql]diff --git a/test/metabase/request/cookies_test.clj b/test/metabase/request/cookies_test.cljindex a1e95bb8026..d26a19396d7 100644--- a/test/metabase/request/cookies_test.clj+++ b/test/metabase/request/cookies_test.clj@@ -16,7 +16,7 @@:same-site :lax:http-only true:path "/"}- (-> (request.cookies/set-session-cookies {} {} {:key session-key, :type :normal} request-time)+ (-> (request.cookies/set-session-cookies {} {} {:id "session-id", :key session-key, :type :normal} request-time)(get-in [:cookies "metabase.SESSION"])))))(testing "should set `Max-Age` if `remember` is true in request"(is (= {:value session-key@@ -24,7 +24,7 @@:http-only true:path "/":max-age 1209600}- (-> (request.cookies/set-session-cookies {:body {:remember true}} {} {:key session-key, :type :normal} request-time)+ (-> (request.cookies/set-session-cookies {:body {:remember true}} {} {:id "session-id", :key session-key, :type :normal} request-time)(get-in [:cookies "metabase.SESSION"])))))(testing "if `MB_SESSION_COOKIES=true` we shouldn't set a `Max-Age`, even if `remember` is true"(is (= {:value session-key@@ -32,12 +32,12 @@:http-only true:path "/"}(mt/with-temporary-setting-values [session-cookies true]- (-> (request.cookies/set-session-cookies {:body {:remember true}} {} {:key session-key, :type :normal} request-time)+ (-> (request.cookies/set-session-cookies {:body {:remember true}} {} {:id "session-id", :key session-key, :type :normal} request-time)(get-in [:cookies "metabase.SESSION"])))))))))(deftest samesite-none-log-warning-test(mt/with-temporary-setting-values [session-cookie-samesite :none]- (let [session {:key (session/generate-session-key)+ (let [session {:id "session-id", :key (session/generate-session-key):type :normal}request-time (t/zoned-date-time "2022-07-06T02:00Z[UTC]")](testing "should log a warning if SameSite is configured to \"None\" and the site is served over an insecure connection."@@ -76,7 +76,7 @@[{"origin" "http://mysite.com"} false]]](testing (format "With headers %s we %s set the 'secure' attribute on the session cookie"(pr-str headers) (if expected "SHOULD" "SHOULD NOT"))- (let [session {:key (random-uuid)+ (let [session {:id "session-id", :key (str (random-uuid)):type :normal}actual (-> (request.cookies/set-session-cookies {:headers headers} {} session (t/zoned-date-time "2022-07-06T02:01Z[UTC]"))(get-in [:cookies "metabase.SESSION" :secure])@@ -90,7 +90,8 @@(def ^:private test-session-key "092797dd-a82a-4748-b393-697d7bb9ab65")(def ^:private test-full-app-embed-session- {:key test-session-key+ {:id "session-id"+ :key test-session-key:anti_csrf_token test-anti-csrf-token:type :full-app-embed})@@ -169,7 +170,7 @@:metabase-session-type :normal:cookies {request.cookies/metabase-session-cookie {:value "session-key"}request.cookies/metabase-session-timeout-cookie {:value "alive"}}}- session {:key session-key+ session {:id "session-id", :key session-key:type :normal}](is (= {:body "some body":cookies {"metabase.TIMEOUT" {:value "alive"@@ -198,7 +199,7 @@:remember "true":cookies {request.cookies/metabase-session-cookie {:value "session-key"}request.cookies/metabase-session-timeout-cookie {:value "alive"}}}- session {:key session-key+ session {:id "session-id", :key session-key:type :normal}](is (= {:body "some body":cookies {"metabase.TIMEOUT" {:value "alive"@@ -217,7 +218,7 @@(mt/with-temporary-setting-values [session-timeout nil](let [now (t/zoned-date-time "2022-01-01T00:00:00.000Z")expires-in-1-hour (t/plus now (t/hours 1))- session {:key (random-uuid)+ session {:id "session-id", :key (str (random-uuid)):type :normal:expires_at expires-in-1-hour}request {}@@ -237,7 +238,7 @@(let [now (t/zoned-date-time "2022-01-01T00:00:00.000Z");; Session expires in 30 days, but max-session-age is 14 daysexpires-in-30-days (t/plus now (t/days 30))- session {:key (random-uuid)+ session {:id "session-id", :key (str (random-uuid)):type :normal:expires_at expires-in-30-days}request {:body {:remember true}} ; Request permanent cookies@@ -257,7 +258,7 @@(mt/with-temporary-setting-values [session-timeout nil](let [now (t/zoned-date-time "2022-01-01T00:00:00.000Z")expires-in-1-day (t/plus now (t/days 1))- session {:key (random-uuid)+ session {:id "session-id", :key (str (random-uuid)):type :normal:expires_at expires-in-1-day}request {:body {:remember true}} ; User requested permanent cookies@@ -275,7 +276,7 @@(mt/with-temp-env-var-value! [:max-session-age "20160"](mt/with-temporary-setting-values [session-timeout nil](let [now (t/zoned-date-time "2022-01-01T00:00:00.000Z")- session {:key (random-uuid)+ session {:id "session-id", :key (str (random-uuid)):type :normal:expires_at nil}request {:body {:remember false}} ; Don't rememberdiff --git a/test/metabase/revisions/api_test.clj b/test/metabase/revisions/api_test.cljindex cb498195ab6..b4722427942 100644--- a/test/metabase/revisions/api_test.clj+++ b/test/metabase/revisions/api_test.clj@@ -543,12 +543,14 @@:model/Dashboard {dashboard-id :id} {:name "A dashboard"}](testing "Reverting a card...";; Create the revision with an extra, unknown field on the card- (revision/push-revision!- {:object (assoc (t2/select-one :model/Card :id card-id) :unknown_field true)- :entity :model/Card- :id card-id- :user-id (mt/user->id :crowberto)- :is-creation? false})+ (t2/insert! :model/Revision+ {:model "Card"+ :model_id card-id+ :user_id (mt/user->id :crowberto)+ :object (assoc (revision/serialize-instance :model/Card card-id (t2/select-one :model/Card :id card-id))+ :unknown_field true)+ :is_creation false+ :is_reversion false});; Update the card to a new version(t2/update! :model/Card {:name "A card with a new name"});; Revert to the saved revision and check that the revert succeeded despite the extra field@@ -557,12 +559,14 @@(is (= "A card" (t2/select-one-fn :name :model/Card :id card-id))))(testing "Reverting a dashboard...";; Create the revision with an extra, unknown field on the dashboard- (revision/push-revision!- {:object (assoc (t2/select-one :model/Dashboard :id dashboard-id) :unknown_field true)- :entity :model/Dashboard- :id dashboard-id- :user-id (mt/user->id :crowberto)- :is-creation? false})+ (t2/insert! :model/Revision+ {:model "Dashboard"+ :model_id dashboard-id+ :user_id (mt/user->id :crowberto)+ :object (assoc (revision/serialize-instance :model/Dashboard dashboard-id (t2/select-one :model/Dashboard :id dashboard-id))+ :unknown_field true)+ :is_creation false+ :is_reversion false});; Update the dashboard to a new version(t2/update! :model/Dashboard {:name "A dashboard with a new name"});; Revert to the saved revision and check that the revert succeeded despite the extra fielddiff --git a/test/metabase/search/api_test.clj b/test/metabase/search/api_test.cljindex 8f7108d4674..9b53979cb27 100644--- a/test/metabase/search/api_test.clj+++ b/test/metabase/search/api_test.clj@@ -1328,7 +1328,7 @@:is-creation? true:object (merge {:id id}(when (= model :model/Card)- {:type "question"}))}))+ {:type :question}))}))(testing "Able to filter by last editor"(let [resp (mt/user-http-request :crowberto :get 200 "search":q search-term@@ -1468,7 +1468,7 @@:is-creation? true:object (merge {:id id}(when (= model :model/Card)- {:type "question"}))}))+ {:type :question}))}))(testing "returns only applicable models"(let [resp (mt/user-http-request :crowberto :get 200 "search" :q search-term :last_edited_at "today":calculate_available_models true)]@@ -1494,7 +1494,7 @@:is-creation? true:object (merge {:id id}(when (= model :model/Card)- {:type "question"}))}))+ {:type :question}))}))(is (= #{"dashboard" "dataset" "metric" "card"}(-> (mt/user-http-request :crowberto :get 200 "search" :q search-term :last_edited_at "today" :last_edited_by (mt/user->id :rasta):calculate_available_models true)@@ -1593,13 +1593,13 @@:id card-id-1:user-id user-id-1:is-creation? true- :object {:id card-id-1 :type "question"}})+ :object {:id card-id-1 :type :question}})(revision/push-revision!{:entity :model/Card:id card-id-2:user-id user-id-2:is-creation? true- :object {:id card-id-2 :type "question"}})+ :object {:id card-id-2 :type :question}})(testing "search result should returns creator_common_name and last_editor_common_name"(is (= #{["card" card-id-1 "Ngoc Khuat" "Ngoc Khuat"];; for user that doesn't have first_name or last_name, should fall backs to emaildiff --git a/test/metabase/search/appdb/specializations/postgres_test.clj b/test/metabase/search/appdb/specializations/postgres_test.cljindex 8bbcf565b2f..e1cf35a9149 100644--- a/test/metabase/search/appdb/specializations/postgres_test.clj+++ b/test/metabase/search/appdb/specializations/postgres_test.clj@@ -12,7 +12,7 @@(deftest batch-upsert-test(with-redefs [t2/query identity](let [query (specialization/batch-upsert! :some-table- [{:a :b}- {:b :c}])]- (is (= [{:a :b} {:b :c}] (:values query)))- (is (= {:a :excluded.a :b :excluded.b} (:do-update-set query))))))+ [{:display_data :a}+ {:legacy_input :b}])]+ (is (= [{:display_data :a} {:legacy_input :b}] (:values query)))+ (is (= {:display_data :excluded.display_data :legacy_input :excluded.legacy_input} (:do-update-set query))))))diff --git a/test/metabase/slackbot/persistence_test.clj b/test/metabase/slackbot/persistence_test.cljindex 2a234f99572..85eb2641449 100644--- a/test/metabase/slackbot/persistence_test.clj+++ b/test/metabase/slackbot/persistence_test.clj@@ -365,25 +365,25 @@(is (= {} (state-of))))(testing "a finished assistant turn's state becomes the baseline"(insert! "assistant" {:queries {"q1" {:database 1}} :todos [{:id "a"}]})- (is (= {:queries {:q1 {:database 1}} :todos [{:id "a"}]}+ (is (= {:queries {"q1" {:database 1}} :todos [{:id "a"}]}(state-of))))(testing "user rows are ignored even when they carry state"(insert! "user" {:queries {"nope" {:database 99}}} :finished nil)- (is (= {:queries {:q1 {:database 1}} :todos [{:id "a"}]}+ (is (= {:queries {"q1" {:database 1}} :todos [{:id "a"}]}(state-of))))(testing "later turns merge in order -- maps merge entry-wise, vectors take the latest"(insert! "assistant" {:queries {"q2" {:database 2}} :todos [{:id "b"}]})- (is (= {:queries {:q1 {:database 1} :q2 {:database 2}} :todos [{:id "b"}]}+ (is (= {:queries {"q1" {:database 1} "q2" {:database 2}} :todos [{:id "b"}]}(state-of))))(testing "errored turns never leak into the baseline"(insert! "assistant" {:todos [{:id "errored"}]} :error "boom")- (is (= {:queries {:q1 {:database 1} :q2 {:database 2}} :todos [{:id "b"}]}+ (is (= {:queries {"q1" {:database 1} "q2" {:database 2}} :todos [{:id "b"}]}(state-of))))(testing "in-flight turns contribute nothing"(insert! "assistant" {:todos [{:id "in-flight"}]} :finished nil)- (is (= {:queries {:q1 {:database 1} :q2 {:database 2}} :todos [{:id "b"}]}+ (is (= {:queries {"q1" {:database 1} "q2" {:database 2}} :todos [{:id "b"}]}(state-of))))(testing "a response deleted from Slack rewinds its state back out"(insert! "assistant" {:todos [{:id "deleted"}]} :deleted? true)- (is (= {:queries {:q1 {:database 1} :q2 {:database 2}} :todos [{:id "b"}]}+ (is (= {:queries {"q1" {:database 1} "q2" {:database 2}} :todos [{:id "b"}]}(state-of)))))))diff --git a/test/metabase/slackbot/query_test.clj b/test/metabase/slackbot/query_test.cljindex 83311e7abf7..7e9c5c138a5 100644--- a/test/metabase/slackbot/query_test.clj+++ b/test/metabase/slackbot/query_test.clj@@ -303,7 +303,7 @@(deftest generate-card-output-display-type-test(testing "generate-card-output returns correct type based on card display"- (let [mock-results {:data {:cols [{:name "x" :base_type :type/Integer}]+ (let [mock-results {:data {:cols [{:name "x" :display_name "x" :base_type :type/Integer}]:rows [[1] [2]]}}](mt/with-dynamic-fn-redefs[slackbot.query/pulse-card-query-results (constantly mock-results)]diff --git a/test/metabase/slackbot/streaming_test.clj b/test/metabase/slackbot/streaming_test.cljindex a2045ed17be..0544129af6d 100644--- a/test/metabase/slackbot/streaming_test.clj+++ b/test/metabase/slackbot/streaming_test.clj@@ -496,7 +496,7 @@(send!)(wait! 2)(testing "the next turn in the same thread picks it up instead of {}"- (is (= {:queries {:q1 {:database 1}}}+ (is (= {:queries {"q1" {:database 1}}}(:state (last @ai-request-calls)))))))))))))(deftest ^:synchronized slackbot-streaming-records-streamed-error-testdiff --git a/test/metabase/sso/ldap_test.clj b/test/metabase/sso/ldap_test.cljindex f8115ea0bc2..3da6e9223e4 100644--- a/test/metabase/sso/ldap_test.clj+++ b/test/metabase/sso/ldap_test.clj@@ -116,7 +116,14 @@(#'default-impl/ldap-groups->mb-group-ids["CN=Accounting,OU=Groups,DC=metabase,DC=com""CN=Shipping,OU=Groups,DC=metabase,DC=com"]- {:group-mappings (sso.settings/ldap-group-mappings)}))))))+ {:first-name-attribute "givenname"+ :last-name-attribute "sn"+ :email-attribute "mail"+ :sync-groups? true+ :user-base "dc=metabase,dc=com"+ :user-filter "(uid={login})"+ :group-base nil+ :group-mappings (sso.settings/ldap-group-mappings)}))))))(deftest valid-group-mapping(testing "Validating that a group mapping DN can contain a forward slash when set as a keyword (#29629)"diff --git a/test/metabase/sync/sync_metadata/fields/sync_metadata_test.clj b/test/metabase/sync/sync_metadata/fields/sync_metadata_test.cljindex b96672a49c5..5134afef79d 100644--- a/test/metabase/sync/sync_metadata/fields/sync_metadata_test.clj+++ b/test/metabase/sync/sync_metadata/fields/sync_metadata_test.clj@@ -319,8 +319,7 @@:semantic_type nil}]](updates-that-will-be-performed!(merge default-metadata- {:id field-id- :base-type :type/Text+ {:base-type :type/Text:effective-type :type/Text})(merge default-metadata{:id field-id@@ -499,14 +498,12 @@(field-user-settings/upsert-user-settings {:id field-id} {:semantic_type :type/Category})(let [updates (updates-that-will-be-performed!(merge default-metadata- {:id field-id- :base-type :type/Text+ {:base-type :type/Text:effective-type :type/Text})(merge default-metadata- {:id field-id- :base-type :type/Integer- :effective-type :type/Integer- :data-sensitivity :PII}))]+ {:id field-id+ :base-type :type/Integer+ :effective-type :type/Integer}))](is (= #{"FieldUserSettings" "Field"} (into #{} (map first) updates)))(is (not-any? (fn [[_ _ changes]] (contains? changes :data_sensitivity)) updates)))))(testing "a user-set label survives a column being dropped and re-created with a different type"diff --git a/test/metabase/sync/sync_metadata/tables_test.clj b/test/metabase/sync/sync_metadata/tables_test.cljindex 8bdd0dd33f1..3fdb3548521 100644--- a/test/metabase/sync/sync_metadata/tables_test.clj+++ b/test/metabase/sync/sync_metadata/tables_test.clj@@ -78,7 +78,7 @@(:engine (mt/db))dbdef(tx/map->TableDefinition {:table-name "user" :table-comment "added comment"}))])- (sync/sync-database! (mt/db) {:sync :schema})+ (sync/sync-database! (mt/db) {:scan :schema})(is (=? {:active true:description "added comment"}(t2/select-one :model/Table (mt/id :user))))))))diff --git a/test/metabase/sync/util_test.clj b/test/metabase/sync/util_test.cljindex a4197e7d532..c23255f9d35 100644--- a/test/metabase/sync/util_test.clj+++ b/test/metabase/sync/util_test.clj@@ -115,7 +115,7 @@(let [process-name (mt/random-name)step-1-name (mt/random-name)step-2-name (mt/random-name)- sync-steps [(sync-util/create-sync-step step-1-name (fn [_] (Thread/sleep 10) {:foo "bar"}))+ sync-steps [(sync-util/create-sync-step step-1-name (fn [_] (Thread/sleep 10) {:total-tables 1}))(sync-util/create-sync-step step-2-name (fn [_] (Thread/sleep 10)))]mock-db (mi/instance :model/Database {:name "test", :id 1, :engine :h2})[results] (:operation-results@@ -133,7 +133,7 @@(is (=? (merge default-task-history {:task process-name, :task_details nil})(fetch-task-history-row process-name))))(testing "step 1 history"- (is (=? (merge default-task-history {:task step-1-name, :task_details {:foo "bar"}})+ (is (=? (merge default-task-history {:task step-1-name, :task_details {:total-tables 1}})(fetch-task-history-row step-1-name))))(testing "step 2 history"(is (=? (merge default-task-history {:task step-2-name, :task_details nil})diff --git a/test/metabase/task_history/models/task_history_test.clj b/test/metabase/task_history/models/task_history_test.cljindex bdbf64038d2..086ea9f4664 100644--- a/test/metabase/task_history/models/task_history_test.clj+++ b/test/metabase/task_history/models/task_history_test.clj@@ -109,24 +109,25 @@(testing "on-success-info"(let [task-name (mt/random-name)](task-history/with-task-history {:task task-name- :task_details {:id 1}+ :task_details {:success 1 :error 0}:on-success-info (fn [info result](testing "info should have task_details, logs, and updated status"- (is (=? {:task_details {:id 1}+ (is (=? {:task_details {:success 1 :error 0}:logs (mt/malli=? vector?):status :success}info)))- (update info :task_details assoc :result result))}+ (update info :task_details assoc :skipped result))}42)(is (= {:status :success- :task_details {:id 1- :result 42}}+ :task_details {:success 1+ :error 0+ :skipped 42}}(t2/select-one [:model/TaskHistory :status :task_details] :task task-name)))))(testing "on-fail-info"(let [task-name (mt/random-name)](u/ignore-exceptions(task-history/with-task-history {:task task-name- :task_details {:id 1}+ :task_details {:success 1 :error 0}:on-fail-info (fn [info e](testing "info should have task_details and updated status"(is (=? {:status :failed@@ -134,7 +135,7 @@:message "test":stacktrace (mt/malli=? :any):ex-data {:reason :test}- :original-info {:id 1}}}+ :original-info {:success 1 :error 0}}}info)))(update info :task_details assoc :reason (ex-message e)))}(throw (ex-info "test" {:reason :test}))))@@ -144,7 +145,7 @@:message "test":stacktrace (mt/malli=? :any):ex-data {:reason "test"}- :original-info {:id 1}+ :original-info {:success 1 :error 0}:reason "test"}}(t2/select-one [:model/TaskHistory :status :task_details] :task task-name)))))))diff --git a/test/metabase/task_history/models/task_run_test.clj b/test/metabase/task_history/models/task_run_test.cljindex 2e6d125ce44..cfc6b3cc429 100644--- a/test/metabase/task_history/models/task_run_test.clj+++ b/test/metabase/task_history/models/task_run_test.clj@@ -222,7 +222,7 @@(testing "sync-operation creates a task run with type :sync"(let [mock-db (mi/instance :model/Database {:name "test" :id 999 :engine :h2})step-name (mt/random-name)- sync-steps [(sync-util/create-sync-step step-name (fn [_] {:done true}))]]+ sync-steps [(sync-util/create-sync-step step-name (fn [_] {:total-tables 1}))]](sync-util/sync-operation :sync-metadata mock-db "Test sync"(sync-util/run-sync-operation "test-sync" mock-db sync-steps))(let [run (t2/select-one :model/TaskRun :entity_type :database :entity_id 999)]@@ -237,7 +237,7 @@(testing "analyze operation creates a task run with type :fingerprint"(let [mock-db (mi/instance :model/Database {:name "test" :id 998 :engine :h2})step-name (mt/random-name)- sync-steps [(sync-util/create-sync-step step-name (fn [_] {:done true}))]]+ sync-steps [(sync-util/create-sync-step step-name (fn [_] {:total-tables 1}))]](sync-util/sync-operation :analyze mock-db "Test analyze"(sync-util/run-sync-operation "test-analyze" mock-db sync-steps))(let [run (t2/select-one :model/TaskRun :entity_type :database :entity_id 998)]@@ -257,8 +257,8 @@(sync-util/sync-operation :sync mock-db "Inner sync"(sync-util/run-sync-operation "inner"mock-db- [(sync-util/create-sync-step inner-step (fn [_] {:inner true}))]))- {:outer true}))]]+ [(sync-util/create-sync-step inner-step (fn [_] {:total-tables 1}))]))+ {:total-tables 1}))]](sync-util/sync-operation :sync mock-db "Outer sync"(sync-util/run-sync-operation "outer" mock-db outer-steps))(is (= 1 (t2/count :model/TaskRun :entity_id 997))diff --git a/test/metabase/test/data.clj b/test/metabase/test/data.cljindex fa02363c8ae..583c36bb552 100644--- a/test/metabase/test/data.clj+++ b/test/metabase/test/data.clj@@ -188,7 +188,7 @@"DEPRECATED: Use Lib to generate MBQL queries instead of hand-rolling legacy MBQL queries in new tests.Like `mbql-query`, but for native queries."- [inner-native-query :- :map]+ [inner-native-query :- :metabase.lib.util/query-like]{:deprecated "0.61.0"}{:database (id):type :native@@ -337,8 +337,8 @@[:+conn-props {:optional true} [:sequential :string]][:-conn-props {:optional true} [:sequential :string]][:+parent {:optional true} :keyword]- [:+fns {:optional true} [:sequential [:function [:=> [:cat :keyword] :any]]]]- [:-fns {:optional true} [:sequential [:function [:=> [:cat :keyword] :any]]]]])+ [:+fns {:optional true} [:sequential [:function [:=> [:cat :keyword] [:or :boolean [:maybe :keyword]]]]]]+ [:-fns {:optional true} [:sequential [:function [:=> [:cat :keyword] [:or :boolean [:maybe :keyword]]]]]]])(mu/defn driver-select :- [:set :keyword]"Select drivers to be tested.diff --git a/test/metabase/test/data/impl.clj b/test/metabase/test/data/impl.cljindex 6a7795f1179..d47fa087598 100644--- a/test/metabase/test/data/impl.clj+++ b/test/metabase/test/data/impl.clj@@ -12,6 +12,7 @@[metabase.test.data.interface :as tx][metabase.util :as u][metabase.util.malli :as mu]+ [metabase.warehouses.schema :as warehouses.schema][methodical.core :as methodical][potemkin :as p][toucan2.core :as t2]@@ -56,7 +57,7 @@does exactly what it suggests."#'get-or-create-default-dataset!)-(mu/defn db :- [:map [:id ::lib.schema.id/database]]+(mu/defn db :- ::warehouses.schema/database[](*db-fn*))@@ -192,7 +193,7 @@(mu/defn do-with-db"Internal impl of [[metabase.test.data/with-db]]."- [db :- [:map [:id ::lib.schema.id/database]]+ [db :- ::warehouses.schema/databasethunk :- fn?](binding [*db-fn* (constantly db)*db-id-fn* (constantly (u/the-id db))@@ -216,7 +217,7 @@(mu/defn database-source-dataset-name :- :string"Get the name of the test dataset this Database was created from, e.g. `test-data`."- [database :- [:map [:settings [:map [:database-source-dataset-name :string]]]]]+ [database :- [:or ::warehouses.schema/database ::warehouses.schema/database.update]](get-in database [:settings :database-source-dataset-name]))(mu/defn the-table-id :- ::lib.schema.id/tablediff --git a/test/metabase/test/data/impl/get_or_create.clj b/test/metabase/test/data/impl/get_or_create.cljindex 05636a567fe..f431e79fcec 100644--- a/test/metabase/test/data/impl/get_or_create.clj+++ b/test/metabase/test/data/impl/get_or_create.clj@@ -88,9 +88,8 @@(mu/defn- add-extra-metadata!"Add extra metadata like Field base-type, etc."- [{:keys [table-definitions], :as _database-definition} :- [:map- [:table-definitions {:optional true} [:maybe [:sequential :map]]]]- db :- :map]+ [{:keys [table-definitions], :as _database-definition} :- tx/DatabaseDefinitionSchema+ db :- :metabase.warehouses.schema/database](doseq [{:keys [table-name], :as table-definition} table-definitions](let [table (delay (or (tx/metabase-instance table-definition db)(throw (Exception. (format "Table '%s' not loaded from definition:\n%s\nFound:\n%s"@@ -158,7 +157,7 @@(:native base-type)(and (map? base-type) (contains? base-type :natives))- (get-in base-type [:natives driver])+ (get-in base-type [:natives (u/qualified-name driver)]):else;; Use fake-sync-database-type to get the type the database reports@@ -423,7 +422,7 @@(mu/defn- create-and-sync-Database!"Add DB object to Metabase DB. Return an instance of `:model/Database`."[driver :- :keyword- {:keys [database-name], :as database-definition} :- [:map [:database-name :string]]]+ {:keys [database-name], :as database-definition} :- tx/DatabaseDefinitionSchema](let [connection-details (tx/dbdef->connection-details driver :db database-definition)db (first (t2/insert-returning-instances! :model/Database(mergediff --git a/test/metabase/test/data/interface.clj b/test/metabase/test/data/interface.cljindex 6975f03e2f5..5e2c0454c39 100644--- a/test/metabase/test/data/interface.clj+++ b/test/metabase/test/data/interface.clj@@ -32,6 +32,7 @@[metabase.util.date-2 :as u.date][metabase.util.log :as log][metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms][metabase.util.random :as u.random][metabase.warehouse-schema.models.field :as field]@@ -63,60 +64,107 @@(p.types/defrecord+ DatabaseDefinition [database-name table-definitions options])-(def ^:private FieldDefinitionSchema+(mr/def ::dataset-value+ "One cell value in a test dataset row, whose type is whatever the driver's test extension knows how to insert."+ [:schema {::mr/deliberately-open true, :description "a test dataset cell value"} :any])++(mr/def ::native-ddl-value+ [:or+ :string :keyword number? :boolean nil?+ [:sequential [:ref ::native-ddl-value]]+ [:ref ::native-ddl-clause]])++(mr/def ::native-ddl-clause[:map {:closed true}- [:field-name ms/NonBlankString]- [:base-type [:or- [:map {:closed true}- [:natives [:map-of :keyword ms/NonBlankString]]]- [:map {:closed true}- [:native ms/NonBlankString]]- ms/FieldType]]- ;; this was added pretty recently (in the 44 cycle) so it might not be supported everywhere. It should work for- ;; drivers using `:sql/test-extensions` and [[metabase.test.data.sql/field-definition-sql]] but you might need to add- ;; support for it elsewhere if you want to use it. It only really matters for testing things that modify test- ;; datasets e.g. [[mt/with-actions-test-data]]- ;; default is nullable- [:not-null? {:optional true} [:maybe :boolean]]- [:unique? {:optional true} [:maybe :boolean]]- [:pk? {:optional true} [:maybe :boolean]]- [:default-expr {:optional true} [:maybe :string]]- [:generated-expr {:optional true} [:maybe :string]]- ;; should we create an index for this field?- [:indexed? {:optional true} [:maybe :boolean]]- [:semantic-type {:optional true} [:maybe ms/FieldSemanticOrRelationType]]- [:effective-type {:optional true} [:maybe ms/FieldType]]- [:coercion-strategy {:optional true} [:maybe ms/CoercionStrategy]]- [:visibility-type {:optional true} [:maybe (into [:enum] field/visibility-types)]]- [:fk {:optional true} [:maybe ms/KeywordOrString]]- [:field-comment {:optional true} [:maybe ms/NonBlankString]]- [:nested-fields {:optional true} [:maybe [:sequential :any]]]])+ [:create-table {:optional true} [:sequential [:ref ::native-ddl-value]]]+ [:with-columns {:optional true} [:sequential [:ref ::native-ddl-value]]]+ [:drop-table {:optional true} [:sequential [:ref ::native-ddl-value]]]+ [:create-index {:optional true} [:sequential [:ref ::native-ddl-value]]]+ [:drop-index {:optional true} [:sequential [:ref ::native-ddl-value]]]+ [:alter-table {:optional true} [:ref ::native-ddl-value]]+ [:add-column {:optional true} [:sequential [:ref ::native-ddl-value]]]+ [:add-constraint {:optional true} [:sequential [:ref ::native-ddl-value]]]+ [:create-view {:optional true} [:ref ::native-ddl-value]]+ [:create-materialized-view {:optional true} [:ref ::native-ddl-value]]+ [:insert-into {:optional true} [:ref ::native-ddl-value]]+ [:columns {:optional true} [:sequential [:ref ::native-ddl-value]]]+ [:values {:optional true} [:sequential [:ref ::native-ddl-value]]]+ [:select {:optional true} [:sequential [:ref ::native-ddl-value]]]+ [:from {:optional true} [:sequential [:ref ::native-ddl-value]]]+ [:join {:optional true} [:sequential [:ref ::native-ddl-value]]]+ [:where {:optional true} [:ref ::native-ddl-value]]+ [:order-by {:optional true} [:sequential [:ref ::native-ddl-value]]]+ [:limit {:optional true} [:ref ::native-ddl-value]]+ [:raw {:optional true} [:ref ::native-ddl-value]]])++(mr/def ::native-ddl-form+ "A single native DDL statement: raw SQL, or a Honey SQL clause map."+ [:or :string [:ref ::native-ddl-clause]])++(def ^:private FieldDefinitionSchema+ [:schema+ {:registry+ {::field-definition+ [:map {:closed true}+ [:field-name ms/NonBlankString]+ [:base-type [:or+ [:map {:closed true}+ [:natives [:map-of :string ms/NonBlankString]]]+ [:map {:closed true}+ [:native ms/NonBlankString]]+ ms/FieldType]]+ ;; this was added pretty recently (in the 44 cycle) so it might not be supported everywhere. It should work for+ ;; drivers using `:sql/test-extensions` and [[metabase.test.data.sql/field-definition-sql]] but you might need to+ ;; add support for it elsewhere if you want to use it. It only really matters for testing things that modify+ ;; test datasets e.g. [[mt/with-actions-test-data]]+ ;; default is nullable+ [:not-null? {:optional true} [:maybe :boolean]]+ [:unique? {:optional true} [:maybe :boolean]]+ [:pk? {:optional true} [:maybe :boolean]]+ [:default-expr {:optional true} [:maybe :string]]+ [:generated-expr {:optional true} [:maybe :string]]+ ;; should we create an index for this field?+ [:indexed? {:optional true} [:maybe :boolean]]+ [:semantic-type {:optional true} [:maybe ms/FieldSemanticOrRelationType]]+ [:effective-type {:optional true} [:maybe ms/FieldType]]+ [:coercion-strategy {:optional true} [:maybe ms/CoercionStrategy]]+ [:visibility-type {:optional true} [:maybe (into [:enum] field/visibility-types)]]+ [:fk {:optional true} [:maybe ms/KeywordOrString]]+ [:field-comment {:optional true} [:maybe ms/NonBlankString]]+ [:collection-type {:optional true} [:maybe ms/FieldType]]+ [:nested-fields {:optional true} [:maybe [:sequential [:ref ::field-definition]]]]]}}+ ::field-definition])(def ^:private ValidFieldDefinition[:and FieldDefinitionSchema (ms/InstanceOfClass FieldDefinition)])+(def TableDefinitionSchema+ "Schema for a table definition, either a `TableDefinition` record or a plain map of the same shape."+ [:map {:closed true}+ [:table-name ms/NonBlankString]+ [:field-definitions [:sequential FieldDefinitionSchema]]+ [:rows [:sequential [:sequential ::dataset-value]]]+ [:table-comment {:optional true} [:maybe ms/NonBlankString]]])+(def ^:private ValidTableDefinition- [:and- [:map {:closed true}- [:table-name ms/NonBlankString]- [:field-definitions [:sequential ValidFieldDefinition]]- [:rows [:sequential [:sequential :any]]]- [:table-comment {:optional true} [:maybe ms/NonBlankString]]]- (ms/InstanceOfClass TableDefinition)])+ [:and TableDefinitionSchema (ms/InstanceOfClass TableDefinition)])++(def DatabaseDefinitionSchema+ "Schema for a database definition, either a `DatabaseDefinition` record or a plain map of the same shape."+ [:map {:closed true}+ [:database-name ms/NonBlankString] ; this must be unique+ [:table-definitions [:sequential TableDefinitionSchema]]+ [:options [:maybe [:map {:closed true}+ [:native-ddl {:optional true} [:sequential ::native-ddl-form]]+ ;; When true, drivers that support it (e.g., MySQL) will disable FK checks during data loading.+ ;; Useful for datasets with self-referencing FKs that need to be inserted in a single batch.+ [:disable-fk-checks {:optional true} :boolean]+ ;; static datasets are not subject to periodic GC+ [:static {:optional true} :boolean]]]]+ [:hash-key {:optional true} [:maybe (ms/InstanceOfClass DatabaseDefinition)]]])(def ^:private ValidDatabaseDefinition- [:and- [:map {:closed true}- [:database-name ms/NonBlankString] ; this must be unique- [:table-definitions [:sequential ValidTableDefinition]]- [:options [:map {:closed true}- [:native-ddl {:optional true} [:sequential :any]]- ;; When true, drivers that support it (e.g., MySQL) will disable FK checks during data loading.- ;; Useful for datasets with self-referencing FKs that need to be inserted in a single batch.- [:disable-fk-checks {:optional true} :boolean]- ;; static datasets are not subject to periodic GC- [:static {:optional true} :boolean]]]]- (ms/InstanceOfClass DatabaseDefinition)])+ [:and DatabaseDefinitionSchema (ms/InstanceOfClass DatabaseDefinition)]);; TODO - this should probably be a protocol instead;; Tech debt issue: #39350@@ -831,7 +879,7 @@[:tuplems/NonBlankString[:sequential DatasetFieldDefinition]- [:sequential [:sequential :any]]])+ [:sequential [:sequential ::dataset-value]]]);; TODO - not sure everything below belongs in this namespace;; Tech debt issue: #39363@@ -850,8 +898,8 @@(apply dataset-table-definition tabledef))([table-name :- ms/NonBlankString- field-definition-maps- rows]+ field-definition-maps :- [:sequential DatasetFieldDefinition]+ rows :- [:sequential [:sequential ::dataset-value]]](map->TableDefinition{:table-name table-name:rows rows@@ -861,11 +909,14 @@"Parse a dataset definition (from a `defdatset` form or EDN file) and return a DatabaseDefinition instance forcomsumption by various test-data-loading methods."([database-name :- ms/NonBlankString- table-definitions]+ table-definitions :- [:sequential DatasetTableDefinition]](dataset-definition database-name table-definitions {}))([database-name :- ms/NonBlankString- table-definitions- options]+ table-definitions :- [:sequential DatasetTableDefinition]+ options :- [:map {:closed true}+ [:native-ddl {:optional true} [:sequential ::native-ddl-form]]+ [:disable-fk-checks {:optional true} :boolean]+ [:static {:optional true} :boolean]]](mu/validate-throw(ms/InstanceOfClass DatabaseDefinition)(map->DatabaseDefinition@@ -950,7 +1001,11 @@(mu/defn edn-dataset-definition"Define a new test dataset using the definition in an EDN file in the `test/metabase/test/data/dataset_definitions/`directory. (Filename should be `dataset-name` + `.edn`.)"- [dataset-name :- ms/NonBlankString options :- map?]+ [dataset-name :- ms/NonBlankString+ options :- [:map {:closed true}+ [:native-ddl {:optional true} [:sequential ::native-ddl-form]]+ [:disable-fk-checks {:optional true} :boolean]+ [:static {:optional true} :boolean]]](let [get-def (delay(let [file-contents (edn/read-string{:eof nil, :readers {'t #'u.date/parse}}@@ -974,10 +1029,21 @@(pretty [_](list `transformed-dataset-definition new-name (pretty/pretty wrapped-definition))))+(def ^:private AnyDatasetDefinition+ "Any of the dataset-definition types [[get-dataset-definition]] knows how to resolve directly (dispatched+ elsewhere, e.g. by tests, are wrapped in one of these before reaching here)."+ [:or+ (ms/InstanceOfClass DatabaseDefinition)+ (ms/InstanceOfClass NativeDatasetDefinition)+ (ms/InstanceOfClass EDNDatasetDefinition)+ (ms/InstanceOfClass TransformedDatasetDefinition)])+(mu/defn transformed-dataset-definition"Create a dataset definition that is a transformation of an some other one, seqentially applying `transform-fns` toit. The results of `transform-fns` are cached."- [new-name :- ms/NonBlankString wrapped-definition & transform-fns]+ [new-name :- ms/NonBlankString+ wrapped-definition :- AnyDatasetDefinition+ & transform-fns :- [:* fn?]](let [transform-fn (apply comp (reverse transform-fns))get-def (delay(transform-fn@@ -996,7 +1062,7 @@(mu/defn transform-dataset-only-tables :- fn?"Create a function for `transformed-dataset-definition` to only keep some subset of Tables from the original datasetdefinition."- [& table-names]+ [& table-names :- [:* ms/NonBlankString]](transform-dataset-update-tabledefs(let [names (set table-names)](fn [tabledefs]@@ -1089,7 +1155,7 @@(mu/defn flattened-dataset-definition"Create a flattened version of `dbdef` by following resolving all FKs and flattening all rows into the table with`table-name`. For use with timeseries databases like Druid."- [dataset-definition+ [dataset-definition :- AnyDatasetDefinitiontable-name :- ms/NonBlankString](transformed-dataset-definition table-name dataset-definition(fn [dbdef]diff --git a/test/metabase/test/data/mbql_query_impl.cljc b/test/metabase/test/data/mbql_query_impl.cljcindex 4d631cd8663..e61cd7c7bc8 100644--- a/test/metabase/test/data/mbql_query_impl.cljc+++ b/test/metabase/test/data/mbql_query_impl.cljc@@ -63,9 +63,19 @@{:arglists '([strategy token-type source-table-symb & tokens])}(fn [strategy token-type & _] [strategy token-type]))+(def ^:private FieldRefOptionValue+ "A `field-ref` option's value while a `$ids`-style form is being built: either the value itself, or a+ not-yet-evaluated s-expression (e.g. an `[[metabase.test.data/id]]` call) that produces it."+ [:or :int :string :keyword :boolean seq? symbol? nil?])+(mu/defn- field-ref- [id-or-name ; can be integer, string, or a s-expression evaluating to one of these.- options :- [:maybe :map]] ; options should already be a map.+ [id-or-name :- [:or :int :string seq? symbol?]+ ;; options should already be a map.+ options :- [:maybe [:map {:closed true}+ [:base-type {:optional true} FieldRefOptionValue]+ [:source-field {:optional true} FieldRefOptionValue]+ [:join-alias {:optional true} FieldRefOptionValue]+ [:temporal-unit {:optional true} FieldRefOptionValue]]]](case (long *mbql-version*)4 [:field id-or-name (not-empty options)]5 [:field (or options {}) id-or-name]))@@ -82,7 +92,10 @@4 x5 y))-(mu/defn- field-ref-update-options [a-ref f & args]+(mu/defn- field-ref-update-options+ [a-ref :- vector?+ f :- ifn?+ & args :- [:* [:or :keyword :metabase.lib.schema.common/non-blank-string]]](field-ref (field-ref->id-or-name a-ref)(apply f (field-ref->opts a-ref) args)))diff --git a/test/metabase/test/data/postgres.clj b/test/metabase/test/data/postgres.cljindex 14fef2d638a..99f2542f7ac 100644--- a/test/metabase/test/data/postgres.clj+++ b/test/metabase/test/data/postgres.clj@@ -99,7 +99,7 @@(into {}(keep (fn [fielddef](when (isa? (:base-type fielddef) :type/JSON)- [(keyword (:field-name fielddef))+ [(:field-name fielddef)(fn [json][::sql.qp/compiled (h2x/cast "json" json)])])))(:field-definitions tabledef)))]@@ -123,7 +123,7 @@tabledef (m/find-first#(= (:table-name %) "json")(:table-definitions dbdef))]- (is (=? [[{:json_bit (mt/malli=? [:tuple [:= :metabase.driver.sql.query-processor/compiled] :any])}]]+ (is (=? [[{"json_bit" (mt/malli=? [:tuple [:= :metabase.driver.sql.query-processor/compiled] :any])}]](into [](map (fn [chunk](into [] (take 1) chunk)))diff --git a/test/metabase/test/data/sql.clj b/test/metabase/test/data/sql.cljindex 6fe90a6eaf0..e9562c18f28 100644--- a/test/metabase/test/data/sql.clj+++ b/test/metabase/test/data/sql.clj@@ -294,7 +294,7 @@(:native base-type)(and (map? base-type) (contains? base-type :natives))- (get-in base-type [:natives driver])+ (get-in base-type [:natives (u/qualified-name driver)])base-type(field-base-type->sql-type driver base-type))diff --git a/test/metabase/test/data/sql_jdbc/load_data.clj b/test/metabase/test/data/sql_jdbc/load_data.cljindex f5e56ef3fd0..e0f0f89fec2 100644--- a/test/metabase/test/data/sql_jdbc/load_data.clj+++ b/test/metabase/test/data/sql_jdbc/load_data.clj@@ -10,6 +10,7 @@[metabase.driver.sql-jdbc.execute :as sql-jdbc.execute][metabase.driver.sql.query-processor :as sql.qp][metabase.lib.schema.common :as lib.schema.common]+ [metabase.query-processor.schema :as qp.schema][metabase.test :as mt][metabase.test.data.interface :as tx][metabase.test.data.sql :as sql.tx]@@ -19,8 +20,7 @@[metabase.util :as u][metabase.util.honey-sql-2 :as h2x][metabase.util.log :as log]- [metabase.util.malli :as mu]- [metabase.util.malli.registry :as mr]))+ [metabase.util.malli :as mu]))(set! *warn-on-reflection* true)@@ -96,7 +96,7 @@[](let [id-counter (atom 0)](map (fn [row]- (assoc row :id (swap! id-counter inc))))))+ (assoc row "id" (swap! id-counter inc))))))(defn maybe-add-ids-xform"Like [[add-ids-xform]], but only adds `:id` to tables that don't have a `:pk` column."@@ -109,8 +109,7 @@"Transform applied by default that converts each row from a vector to a map keyed by `:field-name` from the tabledefinition."[tabledef]- (let [fields-for-insert (mapv (comp keyword :field-name)- (:field-definitions tabledef))]+ (let [fields-for-insert (mapv :field-name (:field-definitions tabledef))](map (fn [row](zipmap fields-for-insert row)))))@@ -121,20 +120,11 @@(ddl.i/format-name driver (u/qualified-name component)))](sql.qp/->honeysql driver (apply h2x/identifier :table components))))-(mr/def ::rf- [:function- [:=> [:cat] :any]- [:=> [:cat :any] :any]- [:=> [:cat :any :any] :any]])--(mr/def ::xform- [:=> [:cat ::rf] ::rf])-(mu/defn- reducible-chunked-rows :- (lib.schema.common/instance-of-class clojure.lang.IReduceInit)- [rows :- [:sequential :any] ; rows is allowed to be empty.+ [rows :- [:sequential [:or [:sequential ::tx/dataset-value] [:map-of :string ::tx/dataset-value]]] ; rows is allowed to be empty.chunk-size :- [:maybe [:int {:min 1}]]- row-xform :- ::xform- chunk-xform :- ::xform]+ row-xform :- ::qp.schema/xform+ chunk-xform :- ::qp.schema/xform](let [xform (comp (map (fn [chunk](into [] row-xform chunk)))chunk-xform)]@@ -160,8 +150,8 @@(mu/defn- reducible-chunks :- (lib.schema.common/instance-of-class clojure.lang.IReduceInit)[driver :- :keyword- dbdef :- [:map [:database-name :string]]- tabledef :- [:map [:table-name :string]]]+ dbdef :- tx/DatabaseDefinitionSchema+ tabledef :- tx/TableDefinitionSchema](let [rows (:rows tabledef)chunk-size (chunk-size driver dbdef tabledef)row-xform (comp@@ -187,9 +177,10 @@(mu/defn do-insert*![driver :- :keyword^java.sql.Connection conn :- (lib.schema.common/instance-of-class java.sql.Connection)- table-identifier- rows- {:keys [transaction?] :or {transaction? true}}]+ table-identifier :- ::h2x/expr+ rows :- [:sequential [:map-of :string ::tx/dataset-value]]+ {:keys [transaction?] :or {transaction? true}} :- [:maybe [:map {:closed true}+ [:transaction? {:optional true} [:maybe :boolean]]]]](let [statements (ddl/insert-rows-dml-statements driver table-identifier rows)];; `set-parameters` might try to look at DB timezone; we don't want to do that while loading the data because the;; DB hasn't been synced yetdiff --git a/test/metabase/test/data/sql_jdbc/load_data_test.clj b/test/metabase/test/data/sql_jdbc/load_data_test.cljindex 893fefb51e0..b0e13761deb 100644--- a/test/metabase/test/data/sql_jdbc/load_data_test.clj+++ b/test/metabase/test/data/sql_jdbc/load_data_test.clj@@ -9,15 +9,15 @@[metabase.test.data.sql-jdbc.load-data :as sql-jdbc.load-data]))(deftest ^:parallel add-ids-xform-test- (is (= [{:name "A", :id 1} {:name "B", :id 2} {:name "C", :id 3} {:name "D", :id 4}]+ (is (= [{"name" "A", "id" 1} {"name" "B", "id" 2} {"name" "C", "id" 3} {"name" "D", "id" 4}](into [](sql-jdbc.load-data/add-ids-xform)- [{:name "A"} {:name "B"} {:name "C"} {:name "D"}]))))+ [{"name" "A"} {"name" "B"} {"name" "C"} {"name" "D"}]))))(deftest ^:parallel reducible-chunked-rows-test(letfn [(reducible-chunks* [chunk-size](#'sql-jdbc.load-data/reducible-chunked-rows- [{:a 1} {:b 2} {:c 3} {:d 4}]+ [{"a" 1} {"b" 2} {"c" 3} {"d" 4}]chunk-size(sql-jdbc.load-data/add-ids-xform)(map (fn [chunk]@@ -25,22 +25,22 @@(assoc row ::chunk chunk))chunk)))))](testing "unchunked"- (is (= [[{:a 1, :id 1, ::chunk [{:a 1, :id 1} {:b 2, :id 2} {:c 3, :id 3} {:d 4, :id 4}]}- {:b 2, :id 2, ::chunk [{:a 1, :id 1} {:b 2, :id 2} {:c 3, :id 3} {:d 4, :id 4}]}- {:c 3, :id 3, ::chunk [{:a 1, :id 1} {:b 2, :id 2} {:c 3, :id 3} {:d 4, :id 4}]}- {:d 4, :id 4, ::chunk [{:a 1, :id 1} {:b 2, :id 2} {:c 3, :id 3} {:d 4, :id 4}]}]]+ (is (= [[{"a" 1, "id" 1, ::chunk [{"a" 1, "id" 1} {"b" 2, "id" 2} {"c" 3, "id" 3} {"d" 4, "id" 4}]}+ {"b" 2, "id" 2, ::chunk [{"a" 1, "id" 1} {"b" 2, "id" 2} {"c" 3, "id" 3} {"d" 4, "id" 4}]}+ {"c" 3, "id" 3, ::chunk [{"a" 1, "id" 1} {"b" 2, "id" 2} {"c" 3, "id" 3} {"d" 4, "id" 4}]}+ {"d" 4, "id" 4, ::chunk [{"a" 1, "id" 1} {"b" 2, "id" 2} {"c" 3, "id" 3} {"d" 4, "id" 4}]}]](into [] (reducible-chunks* nil)))))(testing "chunk size = 5"- (is (= [[{:a 1, :id 1, ::chunk [{:a 1, :id 1} {:b 2, :id 2} {:c 3, :id 3} {:d 4, :id 4}]}- {:b 2, :id 2, ::chunk [{:a 1, :id 1} {:b 2, :id 2} {:c 3, :id 3} {:d 4, :id 4}]}- {:c 3, :id 3, ::chunk [{:a 1, :id 1} {:b 2, :id 2} {:c 3, :id 3} {:d 4, :id 4}]}- {:d 4, :id 4, ::chunk [{:a 1, :id 1} {:b 2, :id 2} {:c 3, :id 3} {:d 4, :id 4}]}]]+ (is (= [[{"a" 1, "id" 1, ::chunk [{"a" 1, "id" 1} {"b" 2, "id" 2} {"c" 3, "id" 3} {"d" 4, "id" 4}]}+ {"b" 2, "id" 2, ::chunk [{"a" 1, "id" 1} {"b" 2, "id" 2} {"c" 3, "id" 3} {"d" 4, "id" 4}]}+ {"c" 3, "id" 3, ::chunk [{"a" 1, "id" 1} {"b" 2, "id" 2} {"c" 3, "id" 3} {"d" 4, "id" 4}]}+ {"d" 4, "id" 4, ::chunk [{"a" 1, "id" 1} {"b" 2, "id" 2} {"c" 3, "id" 3} {"d" 4, "id" 4}]}]](into [] (reducible-chunks* 5)))))(testing "chunk size = 2"- (is (= [[{:a 1, :id 1, ::chunk [{:a 1, :id 1} {:b 2, :id 2}]}- {:b 2, :id 2, ::chunk [{:a 1, :id 1} {:b 2, :id 2}]}]- [{:c 3, :id 3, ::chunk [{:c 3, :id 3} {:d 4, :id 4}]}- {:d 4, :id 4, ::chunk [{:c 3, :id 3} {:d 4, :id 4}]}]]+ (is (= [[{"a" 1, "id" 1, ::chunk [{"a" 1, "id" 1} {"b" 2, "id" 2}]}+ {"b" 2, "id" 2, ::chunk [{"a" 1, "id" 1} {"b" 2, "id" 2}]}]+ [{"c" 3, "id" 3, ::chunk [{"c" 3, "id" 3} {"d" 4, "id" 4}]}+ {"d" 4, "id" 4, ::chunk [{"c" 3, "id" 3} {"d" 4, "id" 4}]}]](into [] (reducible-chunks* 2)))))))(driver/register! ::h2-unchunked, :parent :h2)@@ -69,18 +69,18 @@(#'sql-jdbc.load-data/reducible-chunks driver dbdef tabledef)))](testing ::h2-unchunked;; only one chunk, we took the first 3 rows- (is (= [[{:name "African"}- {:name "American"}- {:name "Artisan"}]]+ (is (= [[{"name" "African"}+ {"name" "American"}+ {"name" "Artisan"}]](chunks ::h2-unchunked))))(testing ::h2-chunked;; many chunks of size 5, we took the first 3 rows from the first 2 chunks.- (is (= [[{:name "African"}- {:name "American"}- {:name "Artisan"}]- [{:name "Bakery"}- {:name "Bar"}- {:name "Beer Garden"}]]+ (is (= [[{"name" "African"}+ {"name" "American"}+ {"name" "Artisan"}]+ [{"name" "Bakery"}+ {"name" "Bar"}+ {"name" "Beer Garden"}]](chunks ::h2-chunked)))))))(driver/register! ::h2-large-chunk-size, :parent :h2)diff --git a/test/metabase/test/generate.clj b/test/metabase/test/generate.cljindex fdab60e1417..0f18a95500a 100644--- a/test/metabase/test/generate.clj+++ b/test/metabase/test/generate.clj@@ -130,7 +130,7 @@(s/def ::kind #{"row/create" "row/update" "row/delete"});; * http_action-(s/def ::template #{{}})+(s/def ::template #{{:method "GET" :url "https://example.com"}});; * pulse(s/def ::row pos-int?)diff --git a/test/metabase/test/http_client.clj b/test/metabase/test/http_client.cljindex 86ffa662561..b225282270d 100644--- a/test/metabase/test/http_client.clj+++ b/test/metabase/test/http_client.clj@@ -11,6 +11,7 @@[malli.core :as mc][medley.core :as m][metabase.config.core :as config]+ [metabase.request.schema :as request.schema][metabase.server.instance :as server.instance][metabase.server.middleware.session :as mw.session][metabase.server.streaming-response :as streaming-response]@@ -284,15 +285,29 @@:patch http/patch:delete http/delete})+(def ^:private RequestOptions+ "The clj-http request options this test client actually uses."+ [:map {:closed true}+ [:headers {:optional true} [:map-of :string [:maybe :string]]]+ [:as {:optional true} :keyword]+ [:content-type {:optional true} [:or :string :keyword]]+ [:cookie-store {:optional true} (ms/InstanceOfClass org.apache.http.client.CookieStore)]+ [:cookies {:optional true} [:map-of :string ::request.schema/cookie-attrs]]+ [:redirect-strategy {:optional true} :keyword]+ [:decompress-body {:optional true} :boolean]+ [:form-params {:optional true} [:map {:closed true}+ [:SAMLResponse :string]+ [:RelayState {:optional true} [:maybe :string]]]]])+(def ^:private ClientParamsMap[:map {:closed true}- [:credentials {:optional true} [:maybe [:or ms/UUIDString map?]]]+ [:credentials {:optional true} [:maybe [:or ms/UUIDString Credentials]]][:method (into [:enum] (keys method->request-fn))][:expected-status {:optional true} [:maybe ms/PositiveInt]][:url ms/NonBlankString]- [:http-body {:optional true} [:maybe [:or map? vector?]]]- [:query-parameters {:optional true} [:maybe map?]]- [:request-options {:optional true} [:maybe map?]]])+ [:http-body {:optional true} [:maybe ms/HttpRequestBody]]+ [:query-parameters {:optional true} [:maybe ms/HttpQueryParams]]+ [:request-options {:optional true} [:maybe RequestOptions]]])(mu/defn- -client;; Since the params for this function can get a little complicated make sure we validate them@@ -381,17 +396,17 @@(str "/"))content-type (get-in request-options [:headers "content-type"] "application/json")](m/deep-merge- {:accept "json"- :headers {"content-type" content-type- @#'mw.session/metabase-session-header (when credentials- (if (map? credentials)- (authenticate credentials)- credentials))}+ {:headers (u/assoc-dissoc {"content-type" content-type}+ @#'mw.session/metabase-session-header+ (when credentials+ (if (map? credentials)+ (authenticate credentials)+ credentials))):query-string (build-query-string query-parameters):remote-addr "127.0.0.1":request-method method:uri (str *url-prefix* url)}- request-options+ (select-keys request-options [:headers :cookies])(build-body-params http-body content-type))))(mu/defn- -mock-client@@ -453,6 +468,7 @@(:url parsed) (update :url url-escape);; un-nest {:request-options {:request-options <my-options>}} => {:request-options <my-options>}(:request-options parsed) (update :request-options :request-options)+ (map? (:http-body parsed)) (update :http-body update-keys u/qualified-name);; convert query parameters into a flat map [{:k :a, :v 1} {:k :b, :v 2} {:k :b, :v 3}] => {:a 1, :b [2 3]}(:query-parameters parsed) (update :query-parameters (fn [query-params](update-vals (->> query-paramsdiff --git a/test/metabase/test/initialize/plugins.clj b/test/metabase/test/initialize/plugins.cljindex 7bcf9d90071..3de69d37931 100644--- a/test/metabase/test/initialize/plugins.clj+++ b/test/metabase/test/initialize/plugins.clj@@ -30,7 +30,7 @@"Loading plugin manifest (from %s) for driver as if it were a real plugin: %s"pathnm))- (yaml/parse-string (slurp manifest))))))))+ (plugins.init/normalize-manifest (yaml/parse-string (slurp manifest)))))))))(defn- driver-parents"Return the set of parents for `driver`. Based on the value of `:metabase.driver/parents` in its `deps.edn`diff --git a/test/metabase/upload/impl_test.clj b/test/metabase/upload/impl_test.cljindex fee348c9562..e1aaa36001d 100644--- a/test/metabase/upload/impl_test.clj+++ b/test/metabase/upload/impl_test.clj@@ -331,7 +331,18 @@auxiliary-sync-steps :nevercsv-file-prefix "example csv file"}:as options}- f :- [:=> [:cat [:map [:id ::lib.schema.id/card]]] :any]]+ :- [:map {:closed true}+ [:table-prefix {:optional true} [:maybe :string]]+ [:collection-id {:optional true} [:maybe ::lib.schema.id/collection]]+ [:grant-permission? {:optional true} [:maybe :boolean]]+ [:uploads-enabled {:optional true} [:maybe :boolean]]+ [:user-id {:optional true} [:maybe ::lib.schema.id/user]]+ [:db-id {:optional true} [:maybe ::lib.schema.id/database]]+ [:auxiliary-sync-steps {:optional true} [:maybe [:enum :asynchronous :synchronous :never]]]+ [:csv-file-prefix {:optional true} [:maybe :string]]+ [:file {:optional true} [:maybe [:fn #(instance? File %)]]]+ [:schema-name {:optional true} [:maybe :string]]]+ f :- fn?]{:pre [(keyword? driver/*driver*)]}(mt/with-discard-model-updates! [:model/Database](t2/update! :model/Database :uploads_enabled true {:uploads_enabled false})diff --git a/test/metabase/users/models/user_test.clj b/test/metabase/users/models/user_test.cljindex 218d0bee8c8..9fce48ccddb 100644--- a/test/metabase/users/models/user_test.clj+++ b/test/metabase/users/models/user_test.clj@@ -77,7 +77,7 @@(t2/delete! :model/User :email new-user-email)))))))(def ^:private default-invitor- {:email "[email protected]", :is_active true, :first_name "Crowberto"})+ {:id 1, :email "[email protected]", :is_active true, :first_name "Crowberto"});; Admins are notified of new joins via `all-admin-recipients`, which excludes admins who haven't yet accepted their;; own invitation (i.e. `last_login` is still nil). These tests therefore construct fresh admin users with explicit@@ -480,7 +480,7 @@(deftest add-attributes-merges-login-and-jwt-attributes-test(testing "add-attributes should add :attributes key with merged login attributes"- (let [user {:login_attributes {"user_attr" "user_value"}+ (let [user {:id 1, :login_attributes {"user_attr" "user_value"}:jwt_attributes {"jwt_attr" "jwt_value"}:email "[email protected]", :type :personal}result (user/add-attributes user)]@@ -491,7 +491,7 @@(deftest add-attributes-handles-nil-login-attributes-test(testing "add-attributes should handle nil login_attributes"- (let [user {:email "[email protected]", :type :personal+ (let [user {:id 1, :email "[email protected]", :type :personal:jwt_attributes {"jwt_attr" "jwt_value"}}result (user/add-attributes user)](is (= {"jwt_attr" "jwt_value"}@@ -499,7 +499,7 @@(deftest add-attributes-handles-empty-login-attributes-test(testing "add-attributes should handle empty login_attributes"- (let [user {:login_attributes {}+ (let [user {:id 1, :login_attributes {}:jwt_attributes {"jwt_attr" "jwt_value"}:email "[email protected]", :type :personal}result (user/add-attributes user)]@@ -508,8 +508,8 @@(deftest add-attributes-user-overrides-jwt-test(testing "add-attributes: user attributes should override jwt attributes with same keys"- (let [user {:login_attributes {"shared_key" "user_value"- "user_only" "user_val"}+ (let [user {:id 1, :login_attributes {"shared_key" "user_value"+ "user_only" "user_val"}:jwt_attributes {"shared_key" "jwt_value""jwt_only" "jwt_val"}:email "[email protected]", :type :personal}@@ -540,7 +540,7 @@(deftest add-attributes-merges-tenant-attributes-test(testing "add-attributes should merge tenant attributes"(mt/with-dynamic-fn-redefs [tenants/login-attributes (constantly {"tenant_attr" "tenant_value"})]- (let [user {:login_attributes {"user_attr" "user_value"}+ (let [user {:id 1, :login_attributes {"user_attr" "user_value"}:email "[email protected]", :type :personal}result (user/add-attributes user)](is (= {"tenant_attr" "tenant_value"@@ -551,7 +551,7 @@(deftest add-attributes-tenant-handles-nil-login-attributes-test(testing "add-attributes with tenant attributes should handle nil login_attributes"(mt/with-dynamic-fn-redefs [tenants/login-attributes (constantly {"tenant_attr" "tenant_value"})]- (let [user {:email "[email protected]", :type :personal}+ (let [user {:id 1, :email "[email protected]", :type :personal}result (user/add-attributes user)](is (= {"tenant_attr" "tenant_value"}(:attributes result)))))))@@ -559,7 +559,7 @@(deftest add-attributes-tenant-handles-empty-login-attributes-test(testing "add-attributes with tenant attributes should handle empty login_attributes"(mt/with-dynamic-fn-redefs [tenants/login-attributes (constantly {"tenant_attr" "tenant_value"})]- (let [user {:login_attributes {}+ (let [user {:id 1, :login_attributes {}:email "[email protected]", :type :personal}result (user/add-attributes user)](is (= {"tenant_attr" "tenant_value"}@@ -568,7 +568,7 @@(deftest add-attributes-handles-nil-tenant-attributes-test(testing "add-attributes should handle nil tenant attributes"(mt/with-dynamic-fn-redefs [tenants/login-attributes (constantly nil)]- (let [user {:login_attributes {"user_attr" "user_value"}+ (let [user {:id 1, :login_attributes {"user_attr" "user_value"}:email "[email protected]", :type :personal}result (user/add-attributes user)](is (= {"user_attr" "user_value"}@@ -577,7 +577,7 @@(deftest add-attributes-handles-both-nil-tenant-and-user-attributes-test(testing "add-attributes should handle both nil tenant and user attributes"(mt/with-dynamic-fn-redefs [tenants/login-attributes (constantly nil)]- (let [user {:email "[email protected]", :type :personal}+ (let [user {:id 1, :email "[email protected]", :type :personal}result (user/add-attributes user)](is (= {}(:attributes result)))))))@@ -588,22 +588,22 @@(doseq [user-type [:api-key :internal]](testing user-type(is (= {}- (:attributes (user/add-attributes {:email "[email protected]"+ (:attributes (user/add-attributes {:id 1, :email "[email protected]":type user-type:login_attributes {"user_attr" "user_value"}:jwt_attributes {"jwt_attr" "jwt_value"}}))))))(testing "throws (in dev) when :type is missing from the user map, so callers can't forget to select it"(is (thrown-with-msg? clojure.lang.ExceptionInfo#"Invalid input"- (user/add-attributes {:email "[email protected]"+ (user/add-attributes {:id 1, :email "[email protected]":login_attributes {"user_attr" "user_value"}})))))))(deftest add-attributes-user-overrides-tenant-test(testing "add-attributes: user attributes should override tenant attributes with same keys"(mt/with-dynamic-fn-redefs [tenants/login-attributes (constantly {"shared_key" "tenant_value""tenant_only" "tenant_val"})]- (let [user {:login_attributes {"shared_key" "user_value"- "user_only" "user_val"}+ (let [user {:id 1, :login_attributes {"shared_key" "user_value"+ "user_only" "user_val"}:email "[email protected]", :type :personal}result (user/add-attributes user)](is (= {"shared_key" "user_value"diff --git a/test/metabase/util/malli/closed_schemas_test.clj b/test/metabase/util/malli/closed_schemas_test.cljnew file mode 100644index 00000000000..183aa5e2b32--- /dev/null+++ b/test/metabase/util/malli/closed_schemas_test.clj@@ -0,0 +1,49 @@+(ns metabase.util.malli.closed-schemas-test+ (:require+ [clojure.test :refer :all]+ [metabase.util.malli :as mu]+ [metabase.util.malli.closed-schemas :as mu.closed-schemas]))++(set! *warn-on-reflection* true)++(deftest ^:parallel findings-without-any-test+ (are [schema kinds] (= kinds (mapv :kind (mu.closed-schemas/findings schema {:any? false})))+ [:map {:closed true} [:a :any]] []+ [:map-of :string :any] []+ [:map [:a :int]] [:open-map]+ [:maybe map?] [:open-map]+ [:map-of :keyword :int] [:keyword-keyed-map-of]+ [:map {:closed true} [:a [:map [:b :int]]]] [:open-map]))++(deftest check-args-test+ (binding [mu.closed-schemas/*enabled* true]+ (is (nil? (mu.closed-schemas/check-args! `f [[:cat :int [:map {:closed true} [:a :int]]]])))+ (is (thrown-with-msg? clojure.lang.ExceptionInfo+ #"The arguments of metabase.util.malli.closed-schemas-test/f reach schemas that do not declare their shape"+ (mu.closed-schemas/check-args! `f [[:cat :int] [:cat [:map [:a :int]]]])))))++(defn- eval-error-message [form]+ (try+ (eval form)+ nil+ (catch Throwable e+ (ex-message (or (ex-cause e) e)))))++(deftest mu-defn-checks-its-args-test+ (binding [mu.closed-schemas/*enabled* true]+ (testing "an open map argument fails when the function is defined"+ (is (re-find #"reach schemas that do not declare their shape"+ (str (eval-error-message `(mu/defn ~'open-arg [~'m :- [:map [:a :int]]] ~'m))))))+ (testing "and so does a destructured map without a schema"+ (is (re-find #"reach schemas that do not declare their shape"+ (str (eval-error-message `(mu/defn ~'destructured [{:keys [~'a]}] ~'a))))))+ (testing "and so does an argument without a schema"+ (is (re-find #"reach schemas that do not declare their shape"+ (str (eval-error-message `(mu/defn ~'unannotated [~'x] ~'x))))))+ (testing "and an :any"+ (is (re-find #"reach schemas that do not declare their shape"+ (str (eval-error-message `(mu/defn ~'any-arg [~'x :- :any] ~'x))))))+ (testing "a return schema is not checked"+ (is (nil? (eval-error-message `(mu/defn ~'open-return :- :any+ [~'m :- [:map {:closed true} [:a :int]]]+ ~'m)))))))diff --git a/test/metabase/util/malli/defn_test.clj b/test/metabase/util/malli/defn_test.cljindex a8d71d30fa1..c536a66690d 100644--- a/test/metabase/util/malli/defn_test.clj+++ b/test/metabase/util/malli/defn_test.clj@@ -24,9 +24,9 @@(are [fn-tail expected] (= expected(#'mu.defn/annotated-docstring (mc/parse mx/SchematizedParams fn-tail)))'(bar- [x :- [:map [:x int?] [:y int?]]]+ [x :- [:map {:closed true} [:x int?] [:y int?]]](str x))- (str "Inputs: [x :- [:map [:x int?] [:y int?]]]\n"+ (str "Inputs: [x :- [:map {:closed true} [:x int?] [:y int?]]]\n"" Return: :any")'(bar@@ -39,7 +39,7 @@" [x :- :int y :- :int])\n"" Return: :int")))-(mu/defn bar [x :- [:map [:x int?] [:y int?]]] (str x))+(mu/defn bar [x :- [:map {:closed true} [:x int?] [:y int?]]] (str x))(mu/defn baz :- [:map [:x int?] [:y int?]] [] {:x "3"})@@ -59,7 +59,7 @@(is (= "Inputs: []\n Return: [:map [:x int?] [:y int?]]"(:doc (meta #'baz))))))-(mu/defn- boo :- :int "something very important to remember goes here" [_x])+(mu/defn- boo :- :int "something very important to remember goes here" [_x :- :int])(mu/defn qux-1 [])(mu/defn qux-2 "Original docstring." [])@@ -78,9 +78,9 @@([] {:type :sized :size 3})([a :- :int] {:type :sized :size a})([a :- :int b :- :int] {:type :sized :size (+ a b)})- ([a b & c :- [:* :int]] {:type :human- :name "Jim"- :address {:street (str (+ a b (apply + c)) " ln")}}))+ ([a :- :int b :- :int & c :- [:* :int]] {:type :human+ :name "Jim"+ :address {:street (str (+ a b (apply + c)) " ln")}}))(deftest ^:parallel mu-defn-docstrings(testing "docstrings are preserved"@@ -121,7 +121,7 @@["Inputs: ([]"" [a :- :int]"" [a :- :int b :- :int]"- " [a b & c :- [:* :int]])"+ " [a :- :int b :- :int & c :- [:* :int]])"" Return: [:multi"" {:dispatch :type}"" [:sized [:map [:type [:= :sized]] [:size int?]]]"@@ -148,7 +148,7 @@[:=> :cat out][:=> [:cat :int] out][:=> [:cat :int :int] out]- [:=> [:cat :any :any [:* :int]] out]])))+ [:=> [:cat :int :int [:* :int]] out]])))(mu/defn- add-ints :- :int^Integer [x :- :int y :- :int]@@ -174,15 +174,16 @@(testing "returns an instrumented fn"(mt/with-dynamic-fn-redefs [mu.fn/instrument-ns? (constantly true)](let [expansion (macroexpand `(mu/defn ~'f :- :int [] "foo"))]- (is (= '(def f- (clojure.core/let- [&f (clojure.core/fn f_AMPERSAND_ [] "foo")]- (clojure.core/fn- mufn- ([]- (try- (clojure.core/->> (&f) (metabase.util.malli.fn/validate-output {:fn-name 'f} :int))- (catch java.lang.Exception error (throw (metabase.util.malli.fn/fixup-stacktrace error))))))))+ (is (= `(~'def ~'f+ (clojure.core/let+ [~'&f (clojure.core/fn ~'f_AMPERSAND_ [] "foo")]+ (~(symbol "metabase.util.malli.closed-schemas" "check-args!") '~(symbol (str *ns*) "f") [:cat])+ (clojure.core/fn+ ~'mufn+ ([]+ (~'try+ (clojure.core/->> (~'&f) (mu.fn/validate-output {:fn-name '~'f} :int))+ (~'catch java.lang.Exception ~'error (throw (mu.fn/fixup-stacktrace ~'error))))))))(deanon-fn-names expansion))))))))(mu/defn- ^:extra-metadata private-foo :- :intdiff --git a/test/metabase/util/malli/fn_test.clj b/test/metabase/util/malli/fn_test.cljindex 3aad6ae4165..060055dfa85 100644--- a/test/metabase/util/malli/fn_test.clj+++ b/test/metabase/util/malli/fn_test.clj@@ -303,8 +303,8 @@(mu/defn- foo :- keyword? [_x :- string?] "bad output")(mu/defn- bar :- keyword?- ([_x :- string? _y] "bad output")- ([_x :- string? _y & _xs] "bad output"))+ ([_x :- string? _y :- :int] "bad output")+ ([_x :- string? _y :- :int & _xs :- [:* :int]] "bad output"))(mu/defn- works? :- keyword? [_x :- string?] :yes)diff --git a/test/metabase/warehouse_schema_overlay/core_test.clj b/test/metabase/warehouse_schema_overlay/core_test.cljindex 8d78a1477e6..812c4a31e32 100644--- a/test/metabase/warehouse_schema_overlay/core_test.clj+++ b/test/metabase/warehouse_schema_overlay/core_test.clj@@ -9,10 +9,10 @@(deftest ^:parallel table-columns-match-the-table-schema-test(testing "the spelled-out column list keeps up with the Table schema it cannot require"- (is (= (set (mu/map-schema-keys :metabase.warehouse-schema.schema/table))+ (is (= (into #{:id :unique_table_helper} (mu/map-schema-keys :metabase.warehouse-schema.schema/table.update))warehouse-schema-overlay/table-columns))))(deftest ^:parallel field-columns-match-the-field-schema-test(testing "the spelled-out column list keeps up with the Field schema it cannot require"- (is (= (set (mu/map-schema-keys :metabase.warehouse-schema.schema/field))+ (is (= (into #{:id :unique_field_helper} (mu/map-schema-keys :metabase.warehouse-schema.schema/field.update))warehouse-schema-overlay/field-columns))))diff --git a/test/metabase/warehouses_rest/api_test.clj b/test/metabase/warehouses_rest/api_test.cljindex 4c1ac0669d5..268fabe5110 100644--- a/test/metabase/warehouses_rest/api_test.clj+++ b/test/metabase/warehouses_rest/api_test.clj@@ -75,7 +75,7 @@(defmethod driver/dbms-version ::test-driver[_ _]- "1.0")+ {:version "1.0"})(defmethod driver/describe-database* ::test-driver[_ _]@@ -579,14 +579,14 @@(with-redefs [driver/can-connect? (constantly true)](is (= nil(:valid (update! 200))))- (let [curr-db (t2/select-one [:model/Database :name :engine :details :is_full_sync], :id db-id)]+ (let [curr-db (t2/select-one [:model/Database :id :name :engine :details :is_full_sync], :id db-id)](is (={:details {:host "localhost", :port 5432, :dbname "fakedb", :user "rastacan"}:engine :h2:name "Cam's Awesome Toucan Database":is_full_sync false:features (driver.u/features :h2 curr-db)}- (into {} curr-db)))))))))))+ (dissoc (into {} curr-db) :id)))))))))))(deftest update-database-test-2(testing "PUT /api/database/:id"diff --git a/test/metabase/xrays/automagic_dashboards/core_test.clj b/test/metabase/xrays/automagic_dashboards/core_test.cljindex 897015cc204..9c4f0a3dfae 100644--- a/test/metabase/xrays/automagic_dashboards/core_test.clj+++ b/test/metabase/xrays/automagic_dashboards/core_test.clj@@ -324,7 +324,7 @@(test-automagic-analysis (t2/select-one :model/Card :id card-id) 2))))))(mu/defn- result-metadata-for-query :- [:maybe [:sequential :map]]- [query :- :map]+ [query :- :metabase.legacy-mbql.schema/Query];; x-ray tests consume legacy-shaped result metadata (persisted card result_metadata format)#_{:clj-kondo/ignore [:deprecated-var]}(qp.metadata/legacy-result-metadata query nil))@@ -1600,7 +1600,7 @@(meta/field-metadata :orders :id))]))))}}{:card {:dataset_query (lib/query meta/metadata-provider (meta/table-metadata :reviews))}}{:card {:dataset_query (lib/query meta/metadata-provider (meta/table-metadata :reviews))}}- {:viz_settings nil}]}+ {}]}{:dataset_query (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))(lib/join (meta/table-metadata :products)))}):dashcards@@ -1620,7 +1620,7 @@(lib/expression "Existing" (lib/- 1 1)))}}{:card {:dataset_query (lib/query meta/metadata-provider (meta/table-metadata :venues))}}{:card {:dataset_query (lib/query meta/metadata-provider (meta/table-metadata :venues))}}- {:viz_settings nil}]}+ {}]}{:dataset_query (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))(lib/expression "TestColumn" (lib/+ 1 1)))}):dashcardsdiff --git a/test/metabase/xrays/automagic_dashboards/interesting_test.clj b/test/metabase/xrays/automagic_dashboards/interesting_test.cljindex 115da281ed3..5b4e1b824e5 100644--- a/test/metabase/xrays/automagic_dashboards/interesting_test.clj+++ b/test/metabase/xrays/automagic_dashboards/interesting_test.clj@@ -87,9 +87,13 @@(deftest ^:parallel field-candidates-with-tablespec-specialization(testing "Test for when both a tablespec and fieldspec are provided in the dimension definition"(let [matching-field {:name "QUANTITY BUT NAME DOES NOT MATTER"+ :display_name "Quantity"+ :base_type :type/*:semantic_type :type/Quantity}- non-matching-field {:name "QUANTITY IS MY NAME, BUT I AM A GENERIC NUMBER"- :semantic_type :type/GenericNumber}+ non-matching-field {:name "QUANTITY IS MY NAME, BUT I AM A DISCOUNT"+ :display_name "Discount"+ :base_type :type/*+ :semantic_type :type/Discount}context {:tables[{:entity_type :entity/GenericTable:fields [matching-field@@ -121,17 +125,23 @@(testing "Tests for when only a fieldspec is provided in the dimension definition.The expectation is a `source` will be provided with populated fields."(let [quantity-field {:name "QUANTITY BUT NAME DOES NOT MATTER"+ :display_name "Quantity"+ :base_type :type/*:semantic_type :type/Quantity}- generic-number-field {:name "QUANTITY IS MY NAME, BUT I AM A GENERIC NUMBER"- :semantic_type :type/GenericNumber}+ generic-number-field {:name "QUANTITY IS MY NAME, BUT I AM A DISCOUNT"+ :display_name "Discount"+ :base_type :type/*+ :semantic_type :type/Discount}another-field {:name "X"- :semantic_type :type/GenericNumber}+ :display_name "X"+ :base_type :type/*+ :semantic_type :type/Discount}context {:source{:fields [quantity-fieldgeneric-number-field]}}quantity-dimension {:field_type [:type/Quantity], :score 100}gt-quantity-dimension {:field_type [:entity/GenericTable :type/Quantity], :score 100}- generic-number-dimension {:field_type [:type/GenericNumber], :score 100}]+ generic-number-dimension {:field_type [:type/Discount], :score 100}](testing "A match occurs when the dimension field_type tablespec and fieldspecmatch the table entity_type and field semantic_type."(is (=? [quantity-field]@@ -153,6 +163,7 @@(testing "Candidate bindings with one field and multiple bindings"(let [field {:base_type :type/Integer:name "QUANTITY"+ :display_name "Quantity":semantic_type :type/Quantity}context {:tables[{:entity_type :entity/GenericTable@@ -175,9 +186,11 @@(testing "Candidate bindings with multiple fields and bindings"(let [nurnies {:base_type :type/Integer:name "Number of Nurnies"+ :display_name "Number of Nurnies":semantic_type :type/Quantity}greebles {:base_type :type/Integer:name "Number of Greebles"+ :display_name "Number of Greebles":semantic_type :type/Quantity}context {:tables[{:entity_type :entity/GenericTable@@ -209,12 +222,15 @@(testing "Candidate bindings with multiple fields and bindings"(let [nurnies {:base_type :type/Integer:name "Number of Nurnies"+ :display_name "Number of Nurnies":semantic_type :type/Quantity}greebles {:base_type :type/Integer:name "Number of Greebles"+ :display_name "Number of Greebles":semantic_type :type/Quantity}- froobs {:base_type :type/Float- :name "A double number field"}+ froobs {:base_type :type/Float+ :name "A double number field"+ :display_name "A double number field"}context {:tables[{:entity_type :entity/GenericTable:fields [nurnies@@ -272,12 +288,15 @@(testing "Perform end-to-end dimension binding with multiple dimensions and fields."(let [nurnies {:base_type :type/Integer:name "Number of Nurnies"+ :display_name "Number of Nurnies":semantic_type :type/Quantity}greebles {:base_type :type/Integer:name "Number of Greebles"+ :display_name "Number of Greebles":semantic_type :type/Quantity}- froobs {:base_type :type/Float- :name "A double number field"}+ froobs {:base_type :type/Float+ :name "A double number field"+ :display_name "A double number field"}context {:tables[{:entity_type :entity/GenericTable:fields [nurnies@@ -296,21 +315,24 @@:score 100:matches [{:base_type :type/Integer:name "Number of Nurnies"+ :display_name "Number of Nurnies":semantic_type :type/Quantity:link nil:field_type [:entity/GenericTable :type/Quantity]:score 100}{:base_type :type/Integer:name "Number of Greebles"+ :display_name "Number of Greebles":semantic_type :type/Quantity:link nil:field_type [:entity/GenericTable :type/Quantity]:score 100}]}"GenericNumber" {:field_type [:entity/GenericTable :type/Number]:score 80- :matches [{:base_type :type/Float- :name "A double number field"- :link nil+ :matches [{:base_type :type/Float+ :name "A double number field"+ :display_name "A double number field"+ :link nil:field_type [:entity/GenericTable :type/Number]:score 80}]}}bindings)))))@@ -331,10 +353,12 @@:fields [{:semantic_type :type/Discount:name "DISCOUNT":effective_type :type/Float+ :display_name "Discount":base_type :type/Float}{:semantic_type :type/Quantity:name "QUANTITY":effective_type :type/Integer+ :display_name "Quantity":base_type :type/Integer}]}:tables [{:entity_type :entity/TransactionTable:fields [{:semantic_type :type/Discount@@ -354,9 +378,9 @@(deftest ^:parallel bind-dimensions-single-field-binding-subtleties-test(testing "Fields are always bound to one and only one dimension."(let [context {:tables [{:entity_type :entity/GenericTable- :fields [{:name "DISCOUNT" :base_type :type/Float}- {:name "QUANTITY" :base_type :type/Float}- {:name "Date" :base_type :type/Date}]}]}+ :fields [{:name "DISCOUNT" :display_name "Discount" :base_type :type/Float}+ {:name "QUANTITY" :display_name "Quantity" :base_type :type/Float}+ {:name "Date" :display_name "Date" :base_type :type/Date}]}]}dimension-defs [{"Date" {:field_type [:entity/GenericTable :type/Date], :score 100}}{"Profit" {:field_type [:entity/GenericTable :type/Float], :score 100}}{"Revenue" {:field_type [:entity/GenericTable :type/Float], :score 100}}@@ -534,9 +558,9 @@:metric-name "AvgDiscount"}{:metric ["sum" ["dimension" "GenericNumber"]], :score 100, :metric-name "Sum"}{:metric ["avg" ["dimension" "GenericNumber"]], :score 100, :metric-name "Avg"}]- total-field (t2/instance :model/Field {:id 1, :name "TOTAL", :base_type :type/Number})- discount-field (t2/instance :model/Field {:id 2, :name "DISCOUNT", :base_type :type/Number})- income-field (t2/instance :model/Field {:id 3, :name "INCOME", :base_type :type/Number})]+ total-field (t2/instance :model/Field {:id 1, :name "TOTAL", :display_name "Total", :base_type :type/Number})+ discount-field (t2/instance :model/Field {:id 2, :name "DISCOUNT", :display_name "Discount", :base_type :type/Number})+ income-field (t2/instance :model/Field {:id 3, :name "INCOME", :display_name "Income", :base_type :type/Number})](testing "When no dimensions are provided, we produce grounded dimensionless metrics"(is (= [{:metric-name "Count":metric-title "Count"diff --git a/test/metabase/xrays/domain_entities/core_test.clj b/test/metabase/xrays/domain_entities/core_test.cljindex bca69a99a4d..55517eb3c93 100644--- a/test/metabase/xrays/domain_entities/core_test.clj+++ b/test/metabase/xrays/domain_entities/core_test.clj@@ -10,7 +10,7 @@(is (= [:field (data/id :venues :price) nil](#'de/mbql-reference (t2/select-one :model/Field :id (data/id :venues :price)))))(is (= [:field "PRICE" {:base-type :type/Integer}]- (#'de/mbql-reference (dissoc (t2/select-one :model/Field :id (data/id :venues :price)) :id)))))+ (#'de/mbql-reference {:name "PRICE", :display_name "Price", :base_type :type/Integer}))))(defn- hydrated-table[table-name]diff --git a/test/metabase/xrays/transforms/core_test.clj b/test/metabase/xrays/transforms/core_test.cljindex 36c36b5fc01..ea9097bae87 100644--- a/test/metabase/xrays/transforms/core_test.clj+++ b/test/metabase/xrays/transforms/core_test.clj@@ -109,9 +109,9 @@(testing "Is the validation of results working?"(is (#'tf/validate-results {"VenuesEnhanced" {:entity (mi/instance:model/Card- {:result_metadata [{:base_type :type/Number, :name "AvgPrice"}- {:base_type :type/Number, :name "MaxPrice"}- {:base_type :type/Number, :name "MinPrice"}]})+ {:result_metadata [{:base_type :type/Number, :name "AvgPrice", :display_name "AvgPrice"}+ {:base_type :type/Number, :name "MaxPrice", :display_name "MaxPrice"}+ {:base_type :type/Number, :name "MinPrice", :display_name "MinPrice"}]}):dimensions {"D1" [:field 1 nil]}}}(first @tf.specs/*transform-specs*))))(testing "... and do we throw if we didn't get what we expected?"diff --git a/test_resources/test_user_key_value_types/cats.edn b/test_resources/test_user_key_value_types/cats.ednindex 9bcbef4c999..07f7925c2ea 100644--- a/test_resources/test_user_key_value_types/cats.edn+++ b/test_resources/test_user_key_value_types/cats.edn@@ -1,2 +1,2 @@[:multi {:dispatch :key}- ["meow" [:map [:value [:maybe :string]]]]]+ ["meow" [:map {:closed true} [:value [:maybe :string]]]]]diff --git a/test_resources/test_user_key_value_types/meow.edn b/test_resources/test_user_key_value_types/meow.ednindex 2643dc03fd6..f092f5e927b 100644--- a/test_resources/test_user_key_value_types/meow.edn+++ b/test_resources/test_user_key_value_types/meow.edn@@ -1,4 +1,4 @@-[:map- [:value [:maybe [:map+[:map {:closed true}+ [:value [:maybe [:map {:closed true}[:key1 {:default "default"} :string][:key2 :int]]]]]diff --git a/test_resources/test_user_key_value_types/other.edn b/test_resources/test_user_key_value_types/other.ednindex df63e23fc8a..c8836c6dd14 100644--- a/test_resources/test_user_key_value_types/other.edn+++ b/test_resources/test_user_key_value_types/other.edn@@ -1,2 +1,2 @@-[:map+[:map {:closed true}[:value :string]]