The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Break requiring resolve cycles (#82011)
Ben Grabow30346d6a64a4
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 43dbd0f726a..84472144430 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -887,7 +887,7 @@metabase.driver.settingsmetabase.driver.sqlmetabase.driver.sql-jdbc- metabase.driver.sql-jdbc.connection+ metabase.driver.sql-jdbc.connection.pool-lockmetabase.driver.sql-jdbc.executemetabase.driver.sql-jdbc.execute.diagnosticmetabase.driver.sql-jdbc.sync@@ -1817,6 +1817,7 @@:api #{metabase.permissions.coremetabase.permissions.initmetabase.permissions.metric+ metabase.permissions.pathmetabase.permissions.schema}:uses #{apiapp-db@@ -1881,6 +1882,7 @@{:team "UX West":api #{metabase.premium-features.apimetabase.premium-features.core+ metabase.premium-features.defenterprisemetabase.premium-features.init}:uses #{analytics.interfaceanalytics.internal-stats@@ -2625,7 +2627,7 @@metabase.sync.field-valuesmetabase.sync.initmetabase.sync.schedules- metabase.sync.task.sync-databases+ metabase.sync.task.sync-databases-triggermetabase.sync.util}:uses #{analytics.interfaceapidiff --git a/enterprise/backend/src/metabase_enterprise/advanced_config/file/databases.clj b/enterprise/backend/src/metabase_enterprise/advanced_config/file/databases.cljindex 3182e16e441..c06d20007e2 100644--- a/enterprise/backend/src/metabase_enterprise/advanced_config/file/databases.clj+++ b/enterprise/backend/src/metabase_enterprise/advanced_config/file/databases.clj@@ -8,6 +8,7 @@[metabase-enterprise.advanced-config.settings :as advanced-config.settings][metabase.driver.util :as driver.u][metabase.sample-data.core :as sample-data]+ [metabase.sync.core :as sync][metabase.util :as u][metabase.util.log :as log][metabase.util.quick-task :as quick-task]))@@ -137,8 +138,7 @@(log/info "Created stub database; skipping sync.")(advanced-config.settings/config-from-file-sync-databases)- (let [sync-database! (requiring-resolve 'metabase.sync.core/sync-database!)]- (quick-task/submit-task! (fn [] (sync-database! db))))+ (quick-task/submit-task! (fn [] (sync/sync-database! db))):else(log/info "Sync on database creation when initializing from file is disabled. Skipping sync."))))))))diff --git a/enterprise/backend/src/metabase_enterprise/sandbox/models/sandbox.clj b/enterprise/backend/src/metabase_enterprise/sandbox/models/sandbox.cljindex e0a813659be..fecdd733cf7 100644--- a/enterprise/backend/src/metabase_enterprise/sandbox/models/sandbox.clj+++ b/enterprise/backend/src/metabase_enterprise/sandbox/models/sandbox.clj@@ -20,6 +20,7 @@[metabase.permissions.core :as perms][metabase.premium-features.core :refer [defenterprise]][metabase.query-processor.error-type :as qp.error-type]+ [metabase.query-processor.preprocess :as qp.preprocess][metabase.query-processor.schema :as qp.schema][metabase.request.core :as request][metabase.util :as u]@@ -46,7 +47,7 @@"Return a mapping of field names to corresponding cols for given table."[table-id](into {} (for [col (request/as-admin- ((requiring-resolve 'metabase.query-processor.preprocess/query->expected-cols)+ (qp.preprocess/query->expected-cols{:database (database/table-id->database-id table-id):type :query:query {:source-table table-id}}))]diff --git a/enterprise/backend/src/metabase_enterprise/sandbox/query_processor/middleware/sandboxing.clj b/enterprise/backend/src/metabase_enterprise/sandbox/query_processor/middleware/sandboxing.cljindex 948788052ff..493ded4fc51 100644--- a/enterprise/backend/src/metabase_enterprise/sandbox/query_processor/middleware/sandboxing.clj+++ b/enterprise/backend/src/metabase_enterprise/sandbox/query_processor/middleware/sandboxing.clj@@ -24,8 +24,10 @@[metabase.lib.schema.util :as lib.schema.util][metabase.lib.walk :as lib.walk][metabase.premium-features.core :as premium-features :refer [defenterprise]]+ [metabase.query-processor :as qp][metabase.query-processor.error-type :as qp.error-type][metabase.query-processor.pipeline :as qp.pipeline]+ [metabase.query-processor.preprocess :as qp.preprocess][metabase.query-processor.util.persisted-cache :as qp.persisted][metabase.request.core :as request][metabase.util :as u]@@ -148,15 +150,14 @@(try(lib/without-cleaning(fn []- (let [preprocess (requiring-resolve 'metabase.query-processor.preprocess/preprocess)]- (request/as-admin- ;; preprocessing normally loses metadata attached to the last stage, since legacy MBQL syntax does not- ;; support it and preprocessing roundtrips to legacy and back a few times... to make sure it's preserved,- ;; append an extra dummy stage before preprocessing and then toss it when we're done.- (-> query- lib/append-stage- preprocess- (update :stages pop))))))+ (request/as-admin+ ;; preprocessing normally loses metadata attached to the last stage, since legacy MBQL syntax does not+ ;; support it and preprocessing roundtrips to legacy and back a few times... to make sure it's preserved,+ ;; append an extra dummy stage before preprocessing and then toss it when we're done.+ (-> query+ lib/append-stage+ qp.preprocess/preprocess+ (update :stages pop)))))(catch Throwable e(throw (ex-info (tru "Error preprocessing query when applying Sandbox: {0}" (ex-message e)){:query query}@@ -204,8 +205,7 @@;; to a custom handler, and we don't want to accidentally terminate the stream here!(binding [qp.pipeline/*result* qp.pipeline/default-result-handler](request/as-admin- ((requiring-resolve 'metabase.query-processor/process-query)- query)))]+ (qp/process-query query)))](when-not (= (:status result) :completed)(throw (ex-info "Error running query to determine metadata"{:query query, :result result})))diff --git a/frontend/src/metabase-lib/v1/parameters/utils/template-tags.ts b/frontend/src/metabase-lib/v1/parameters/utils/template-tags.tsindex 86dc3f9ac02..d510acbdd53 100644--- a/frontend/src/metabase-lib/v1/parameters/utils/template-tags.ts+++ b/frontend/src/metabase-lib/v1/parameters/utils/template-tags.ts@@ -65,7 +65,7 @@ export function getTemplateTagParameter(};}-// NOTE: this should mirror `template-tag-parameters` in src/metabase/queries/models/card.clj+// NOTE: this should mirror `template-tag-parameters` in src/metabase/query_processor/card.clj// If this function moves you should update the comment that links to this oneexport function getTemplateTagParameters(tags: TemplateTag[],diff --git a/src/metabase/actions/schema.clj b/src/metabase/actions/schema.cljindex 5d9506c01d1..a2be7f1bb31 100644--- a/src/metabase/actions/schema.clj+++ b/src/metabase/actions/schema.clj@@ -1,5 +1,6 @@(ns metabase.actions.schema(:require+ [metabase.actions.http-action :as http-action][metabase.lib-be.schema :as lib-be.schema][metabase.lib.core :as lib][metabase.lib.schema.common :as lib.schema.common]@@ -94,7 +95,7 @@[:fn{:error/fn (fn [_ _](deferred-tru "must be a valid json-query, something like ''.item.title''"))}- #((requiring-resolve 'metabase.actions.http-action/apply-json-query) {} %)]])+ #(http-action/apply-json-query {} %)]])(mr/def ::http-action.template[:map {:closed true}diff --git a/src/metabase/analytics/prometheus.clj b/src/metabase/analytics/prometheus.cljindex 26e1df4916e..eab04887eb6 100644--- a/src/metabase/analytics/prometheus.clj+++ b/src/metabase/analytics/prometheus.clj@@ -15,6 +15,8 @@[jvm-alloc-rate-meter.core :as alloc-rate-meter][jvm-hiccup-meter.core :as hiccup-meter][metabase.analytics-interface.core :as analytics.interface]+ ;; We should not be using specific driver implementations+ [metabase.driver.sql-jdbc.connection.pool-lock :as pool-lock][metabase.util :as u][metabase.util.i18n :refer [trs]][metabase.util.log :as log]@@ -135,20 +137,18 @@arr))(defn- conn-pool-bean-diag-info [acc ^ObjectName jmx-bean]- ;; We should not be using specific driver implementations- (let [pool-var (requiring-resolve 'metabase.driver.sql-jdbc.connection/pool-cache-key->connection-pool)]- ;; Using this `locking` is non-obvious but absolutely required to avoid the deadlock inside c3p0 implementation. The- ;; act of JMX attribute reading first locks a DynamicPooledDataSourceManagerMBean object, and then a- ;; PoolBackedDataSource object. Conversely, the act of creating a pool (with- ;; com.mchange.v2.c3p0.DataSources/pooledDataSource) first locks PoolBackedDataSource and then- ;; DynamicPooledDataSourceManagerMBean. We have to lock a common monitor (which `pool-cache-key->connection-pool` is)- ;; to prevent the deadlock. Hopefully.- ;; Issue against c3p0: https://github.com/swaldman/c3p0/issues/95- (locking @pool-var- (let [bean-id (.getCanonicalName jmx-bean)- props [:numConnections :numIdleConnections :numBusyConnections- :minPoolSize :maxPoolSize :numThreadsAwaitingCheckoutDefaultUser]]- (assoc acc (jmx/read bean-id :dataSourceName) (jmx/read bean-id props))))))+ ;; Using this `locking` is non-obvious but absolutely required to avoid the deadlock inside c3p0 implementation. The+ ;; act of JMX attribute reading first locks a DynamicPooledDataSourceManagerMBean object, and then a+ ;; PoolBackedDataSource object. Conversely, the act of creating a pool (with+ ;; com.mchange.v2.c3p0.DataSources/pooledDataSource) first locks PoolBackedDataSource and then+ ;; DynamicPooledDataSourceManagerMBean. We have to lock a common monitor (which `pool-lock/monitor` is)+ ;; to prevent the deadlock. Hopefully.+ ;; Issue against c3p0: https://github.com/swaldman/c3p0/issues/95+ (locking pool-lock/monitor+ (let [bean-id (.getCanonicalName jmx-bean)+ props [:numConnections :numIdleConnections :numBusyConnections+ :minPoolSize :maxPoolSize :numThreadsAwaitingCheckoutDefaultUser]]+ (assoc acc (jmx/read bean-id :dataSourceName) (jmx/read bean-id props)))))(defn connection-pool-info"Builds a map of info about the current c3p0 connection pools managed by this Metabase instance."diff --git a/src/metabase/api/docs.clj b/src/metabase/api/docs.cljindex 010f891977c..ad77928d72c 100644--- a/src/metabase/api/docs.clj+++ b/src/metabase/api/docs.clj@@ -3,83 +3,25 @@(:require[clojure.java.io :as io][clojure.string :as str]- [clojure.walk :as walk][compojure.core :refer [GET]]+ [metabase.api.docs.regenerate][metabase.api.open-api :as open-api][metabase.api.util.handlers :as handlers][metabase.config.core :as config][metabase.util.json :as json][metabase.util.log :as log]+ [potemkin :as p][ring.middleware.content-type :as content-type][ring.util.response :as response]))(set! *warn-on-reflection* true)-(def openapi-file-path- "Path to the local OpenAPI specification file."- "resources/openapi/openapi.json")--(defn- sort-keys- "Sort maps and sets alphabetically to reduce diff noise on openapi.json"- [data]- (walk/postwalk- (fn [x]- (cond- (map? x) (into (sorted-map) x)- (set? x) (into (sorted-set) x)- :else x))- data))--(defn write-openapi-spec-to-file!- "Generate and write the OpenAPI specification to a local file.- Takes the root handler and generates the complete OpenAPI spec, writing it to [[openapi-file-path]]."- [root-handler]- (try- (let [spec (merge- (open-api/root-open-api-object root-handler)- {:servers [{:url ""- :description "Metabase API"}]})- file (io/file openapi-file-path)]- ;; Create parent directory if it doesn't exist- (when-let [parent-dir (.getParentFile file)]- (.mkdirs parent-dir))- (json/encode-to (sort-keys spec) (io/writer file) {:pretty true})- (log/info "OpenAPI specification written to" openapi-file-path))- (catch Throwable e- (log/errorf "Failed to write OpenAPI specification to file: %s" (ex-message e)))))--(defonce ^:private openapi-regen-state- (atom {:executing? false- :needs-regen? false}))--(def ^:private debounce-delay-ms- "Delay in milliseconds before starting regeneration to allow multiple rapid requests to coalesce."- 200)--(defn request-spec-regeneration!- "Request OpenAPI spec regeneration. Multiple rapid requests are coalesced into one execution.- Uses a debounce delay to batch requests that arrive in quick succession (e.g., when re-evaling a file with multiple endpoints)."- [routes]- (swap! openapi-regen-state assoc :needs-regen? true)-- (when-not (:executing? @openapi-regen-state)- (swap! openapi-regen-state assoc :executing? true)- (future- (try- ;; Wait for debounce period to let multiple requests coalesce- (Thread/sleep ^Long debounce-delay-ms)- (loop []- (when (:needs-regen? @openapi-regen-state)- (swap! openapi-regen-state assoc :needs-regen? false)- (log/info "Regenerating OpenAPI specification...")- (write-openapi-spec-to-file! routes)- (log/info "OpenAPI specification regenerated successfully")- ;; Check if more requests came in while we were working- (recur)))- (catch Throwable e- (log/errorf "Error regenerating OpenAPI specification: %s" (ex-message e)))- (finally- (swap! openapi-regen-state assoc :executing? false))))))+;; kept on this namespace so existing callers (metabase.api-routes.cmd) are unaffected by the split+(p/import-vars+ [metabase.api.docs.regenerate+ openapi-file-path+ request-spec-regeneration!+ write-openapi-spec-to-file!])(defn- read-openapi-spec-from-file"Read the OpenAPI specification from the local file.diff --git a/src/metabase/api/docs/regenerate.clj b/src/metabase/api/docs/regenerate.cljnew file mode 100644index 00000000000..1e308c9e0d4--- /dev/null+++ b/src/metabase/api/docs/regenerate.clj@@ -0,0 +1,80 @@+(ns metabase.api.docs.regenerate+ "Writing `openapi.json` to disk, and the debounced scheduler that requests a rewrite.++ Split out of [[metabase.api.docs]] so that [[metabase.api.macros]] can name+ [[request-spec-regeneration!]] directly: `metabase.api.docs` reaches `metabase.api.macros` through+ [[metabase.api.util.handlers]], and nothing here does."+ (:require+ [clojure.java.io :as io]+ [clojure.walk :as walk]+ [metabase.api.open-api :as open-api]+ [metabase.util.json :as json]+ [metabase.util.log :as log]))++(set! *warn-on-reflection* true)++(def openapi-file-path+ "Path to the local OpenAPI specification file."+ "resources/openapi/openapi.json")++(defn- sort-keys+ "Sort maps and sets alphabetically to reduce diff noise on openapi.json"+ [data]+ (walk/postwalk+ (fn [x]+ (cond+ (map? x) (into (sorted-map) x)+ (set? x) (into (sorted-set) x)+ :else x))+ data))++(defn write-openapi-spec-to-file!+ "Generate and write the OpenAPI specification to a local file.+ Takes the root handler and generates the complete OpenAPI spec, writing it to [[openapi-file-path]]."+ [root-handler]+ (try+ (let [spec (merge+ (open-api/root-open-api-object root-handler)+ {:servers [{:url ""+ :description "Metabase API"}]})+ file (io/file openapi-file-path)]+ ;; Create parent directory if it doesn't exist+ (when-let [parent-dir (.getParentFile file)]+ (.mkdirs parent-dir))+ (json/encode-to (sort-keys spec) (io/writer file) {:pretty true})+ (log/info "OpenAPI specification written to" openapi-file-path))+ (catch Throwable e+ (log/errorf "Failed to write OpenAPI specification to file: %s" (ex-message e)))))++(defonce ^:private openapi-regen-state+ (atom {:executing? false+ :needs-regen? false}))++(def ^:private debounce-delay-ms+ "Delay in milliseconds before starting regeneration to allow multiple rapid requests to coalesce."+ 200)++(defn request-spec-regeneration!+ "Request OpenAPI spec regeneration. Multiple rapid requests are coalesced into one execution.+ Uses a debounce delay to batch requests that arrive in quick succession (e.g., when re-evaling a file with multiple endpoints)."+ [routes]+ (swap! openapi-regen-state assoc :needs-regen? true)++ (when-not (:executing? @openapi-regen-state)+ (swap! openapi-regen-state assoc :executing? true)+ (future+ (try+ ;; Wait for debounce period to let multiple requests coalesce+ (Thread/sleep ^Long debounce-delay-ms)+ (loop []+ (when (:needs-regen? @openapi-regen-state)+ (swap! openapi-regen-state assoc :needs-regen? false)+ (log/info "Regenerating OpenAPI specification...")+ (write-openapi-spec-to-file! routes)+ (log/info "OpenAPI specification regenerated successfully")+ ;; Check if more requests came in while we were working+ (recur)))+ (catch Throwable e+ (log/errorf "Error regenerating OpenAPI specification: %s" (ex-message e)))+ (finally+ (swap! openapi-regen-state assoc :executing? false))))))diff --git a/src/metabase/api/macros.clj b/src/metabase/api/macros.cljindex 3071ce55d26..ea041a2a8ca 100644--- a/src/metabase/api/macros.clj+++ b/src/metabase/api/macros.clj@@ -27,6 +27,7 @@[malli.util][medley.core :as m][metabase.api.common.internal]+ [metabase.api.docs.regenerate :as docs.regenerate][metabase.api.macros.defendpoint.closed-schemas :as closed-schemas][metabase.api.macros.defendpoint.open-api][metabase.api.macros.scope]@@ -819,8 +820,7 @@(when (config/config-bool :mb-enable-openapi-auto-regen)(try(events/publish-event! :event/api-handler-update- {:api.docs/request-rebuild- (requiring-resolve 'metabase.api.docs/request-spec-regeneration!)})+ {:api.docs/request-rebuild #'docs.regenerate/request-spec-regeneration!})(catch Throwable e(log/debugf "Failed to publish api-handler-update event: %s" (ex-message e)))))))diff --git a/src/metabase/app_db/custom_migrations.clj b/src/metabase/app_db/custom_migrations.cljindex 224b44e84cd..0312dcc0687 100644--- a/src/metabase/app_db/custom_migrations.clj+++ b/src/metabase/app_db/custom_migrations.clj@@ -28,9 +28,9 @@[metabase.app-db.custom-migrations.pulse-to-notification :as pulse-to-notification][metabase.app-db.custom-migrations.reserve-at-symbol-user-attributes :as reserve-at-symbol-user-attributes][metabase.app-db.custom-migrations.util :as custom-migrations.util]+ [metabase.app-db.quartz][metabase.app-db.setting :as mdb.setting][metabase.config.core :as config]- [metabase.task.bootstrap][metabase.util.date-2 :as u.date][metabase.util.encryption :as encryption][metabase.util.honey-sql-2 :as h2x]@@ -1760,7 +1760,7 @@;; on migrate up:;; - migrate alerts from pulse table to notification table;; - And then on startup new send notification triggers are created by running-;; [[metabase.notification.task.send/init-send-notification-triggers!]]+;; [[metabase.notification.task.send-trigger/init-send-notification-triggers!]](define-migration MigrateAlertToNotification(pulse-to-notification/migrate-alerts!))diff --git a/src/metabase/app_db/data_source.clj b/src/metabase/app_db/data_source.cljindex 227577ed2b7..35f41ce865a 100644--- a/src/metabase/app_db/data_source.clj+++ b/src/metabase/app_db/data_source.clj@@ -6,6 +6,7 @@[clojure.string :as str][metabase.app-db.spec :as mdb.spec][metabase.app-db.update-h2 :as update-h2]+ [metabase.auth-provider.core :as auth-provider][metabase.config.core :as config][metabase.connection-pool :as connection-pool][metabase.util.log :as log]@@ -25,10 +26,10 @@(defn- renew-azure-managed-identity-password[client-id](let [{:keys [access_token expires_in]}- ((requiring-resolve 'metabase.auth-provider.core/fetch-auth) :azure-managed-identity nil {:azure-managed-identity-client-id client-id})]+ (auth-provider/fetch-auth :azure-managed-identity nil {:azure-managed-identity-client-id client-id})]{:password access_token:expiry (+ (*current-millis*) (* (- (parse-long expires_in)- @(requiring-resolve 'metabase.auth-provider.core/azure-auth-token-renew-slack-seconds))+ auth-provider/azure-auth-token-renew-slack-seconds)1000))}))(defn- ensure-azure-managed-identity-passworddiff --git a/src/metabase/app_db/quartz.clj b/src/metabase/app_db/quartz.cljnew file mode 100644index 00000000000..4fc65070396--- /dev/null+++ b/src/metabase/app_db/quartz.clj@@ -0,0 +1,29 @@+(ns metabase.app-db.quartz+ "Quartz's `ConnectionProvider` for the application DB."+ (:require+ [metabase.app-db.connection :as mdb.connection]+ [metabase.task.bootstrap :as task.bootstrap]))++(set! *warn-on-reflection* true)++;; Custom `ConnectionProvider` implementation that uses a dedicated connection pool for the application DB to provide+;; connections.+(defrecord ^:private ConnectionProvider []+ org.quartz.utils.ConnectionProvider+ (initialize [_])+ (getConnection [_]+ ;; get a connection from the dedicated Quartz connection pool. Quartz will close it (i.e., return it to the pool)+ ;; when it's done.+ ;;+ ;; very important! Fetch a new connection from the connection pool rather than reusing a Connection already bound+ ;; to the calling thread (e.g. toucan2's *current-connectable*) -- Quartz manages the connection's whole+ ;; lifecycle (setAutoCommit/commit/rollback/close), and its cluster locking relies on commit/rollback to release+ ;; row locks on the QRTZ_LOCKS table, so it must never share a connection with an outer transaction.+ ;;+ ;; the pool is separate from the main application DB pool so that a Quartz operation triggered by a thread inside+ ;; a `with-transaction` block can't deadlock when application code has saturated the main pool.+ (task.bootstrap/intercept-connection (.getConnection (mdb.connection/quartz-data-source))))+ (shutdown [_]))++(when-not *compile-files*+ (System/setProperty "org.quartz.dataSource.db.connectionProvider.class" (.getName ConnectionProvider)))diff --git a/src/metabase/auth_provider/impl.clj b/src/metabase/auth_provider/impl.cljindex d5c94f1726b..89ce6f81227 100644--- a/src/metabase/auth_provider/impl.clj+++ b/src/metabase/auth_provider/impl.clj@@ -1,6 +1,6 @@(ns metabase.auth-provider.impl(:require- [metabase.premium-features.core :refer [defenterprise]]))+ [metabase.premium-features.defenterprise :refer [defenterprise]]))(def azure-auth-token-renew-slack-seconds"How many seconds before expiry we should prefer renewal.diff --git a/src/metabase/channel/email/result_attachment.clj b/src/metabase/channel/email/result_attachment.cljindex de7a7ec05bd..24f2f87856e 100644--- a/src/metabase/channel/email/result_attachment.clj+++ b/src/metabase/channel/email/result_attachment.clj@@ -2,6 +2,7 @@(:require[clojure.java.io :as io][metabase.channel.render.util :as render.util]+ [metabase.channel.shared :as channel.shared][metabase.driver :as driver][metabase.driver.util :as driver.u][metabase.lib.schema.id :as lib.schema.id]@@ -101,8 +102,7 @@(when (and (or (:include_csv card) (:include_xls card))(pos-int? (:row_count result))(not= (perms/download-perms-level (:dataset_query card) creator-id) :no))- (let [maybe-realize-data-rows (requiring-resolve 'metabase.channel.shared/maybe-realize-data-rows)- result (:result (maybe-realize-data-rows part))+ (let [result (:result (channel.shared/maybe-realize-data-rows part));; the dashboard-level title override (incl. the visualizer's) names the attachment, matching the card's;; displayed title; falls back to the card's own name (see [[render.util/dashcard-title]]).filename-prefix (render.util/dashcard-title card dashcard)]diff --git a/src/metabase/collections/models/collection/root.clj b/src/metabase/collections/models/collection/root.cljindex 43d373181ce..510c86c3414 100644--- a/src/metabase/collections/models/collection/root.clj+++ b/src/metabase/collections/models/collection/root.clj@@ -2,7 +2,7 @@(:require[medley.core :as m][metabase.models.interface :as mi]- [metabase.permissions.core :as perms]+ [metabase.permissions.path :as permissions.path][metabase.premium-features.core :as premium-features][metabase.util :as u][metabase.util.i18n :refer [tru]]@@ -37,8 +37,8 @@(not (premium-features/enable-snippet-collections?)))#{}#{((case read-or-write- :read perms/collection-read-path- :write perms/collection-readwrite-path) collection)}))+ :read permissions.path/collection-read-path+ :write permissions.path/collection-readwrite-path) collection)}))(def ^RootCollection root-collection"Special placeholder object representing the Root Collection, which isn't really a real Collection."diff --git a/src/metabase/dashboards/models/dashboard.clj b/src/metabase/dashboards/models/dashboard.cljindex 3a1b5dbece5..08d568cb202 100644--- a/src/metabase/dashboards/models/dashboard.clj+++ b/src/metabase/dashboards/models/dashboard.clj@@ -27,6 +27,7 @@[metabase.search.core :as search][metabase.settings.core :as setting][metabase.staleness.core :as staleness]+ [metabase.sync.field-values :as sync.field-values][metabase.util :as u][metabase.util.embed :refer [maybe-populate-initially-published-at]][metabase.util.honey-sql-2 :as h2x]@@ -225,7 +226,7 @@(log/info "Referenced Fields in Dashboard params have changed: Was:" old-param-field-ids"Is Now:" new-param-field-ids"Newly Added:" newly-added-param-field-ids)- ((requiring-resolve 'metabase.sync.field-values/update-field-values-for-on-demand-dbs!) newly-added-param-field-ids))))+ (sync.field-values/update-field-values-for-on-demand-dbs! newly-added-param-field-ids))))(defn add-dashcards!"Add Cards to a Dashboard.@@ -368,31 +369,9 @@(cond-> dashboard(collections/remote-synced-collection? parent-collection-id) collections/check-non-remote-synced-dependencies))))-(def ^:private ParamWithMapping- [:map- [:id ms/NonBlankString]- [:name ms/NonBlankString]- [:mappings [:maybe [:set ::parameters.schema/parameter-mapping-with-dashcard]]]])--(mu/defn dashboard->resolved-params :- [:map-of ms/NonBlankString ParamWithMapping]- "Return map of Dashboard parameter key -> param with resolved `:mappings` (see the `:resolved-params` hydration- below for an example). Callers that only need the mappings (e.g. the QP) can pass slim dashcards instead of paying- for the full hydration."- [dashboard :- [:map- [:parameters [:maybe [:sequential :map]]]- [:dashcards [:maybe [:sequential [:map- [:parameter_mappings [:maybe [:sequential :map]]]]]]]]]- (let [param-key->mappings (apply- merge-with set/union- (for [dashcard (:dashcards dashboard)- param (:parameter_mappings dashcard)]- {(:parameter_id param) #{(assoc param :dashcard dashcard)}}))]- (into {} (for [{param-key :id, :as param} (:parameters dashboard)]- [(u/qualified-name param-key) (assoc param :mappings (get param-key->mappings param-key))]))))-(methodical/defmethod t2/batched-hydrate [:model/Dashboard :resolved-params]"Return map of Dashboard parameter key -> param with resolved `:mappings`.- (dashboard->resolved-params (t2/select-one Dashboard :id 62))+ (params/dashboard->resolved-params (t2/select-one Dashboard :id 62));; ->{\"ee876336\" {:name \"Category Name\":slug \"category_name\"@@ -412,7 +391,7 @@:target [:dimension [:field-id 264]]}}}}"[_model k dashboards](let [dashboards-with-cards (t2/hydrate dashboards [:dashcards :card :series])]- (map #(assoc %1 k %2) dashboards (map dashboard->resolved-params dashboards-with-cards))))+ (map #(assoc %1 k %2) dashboards (map params/dashboard->resolved-params dashboards-with-cards))))(defmethod mi/exclude-internal-content-hsql :model/Dashboard[_model & {:keys [table-alias]}]diff --git a/src/metabase/driver/init.clj b/src/metabase/driver/init.cljindex 385645cacf1..993852c6f14 100644--- a/src/metabase/driver/init.clj+++ b/src/metabase/driver/init.clj@@ -15,4 +15,5 @@[metabase.driver.settings];; for the `:sql-jdbc` implementation of [[metabase.driver/incorporate-ssh-tunnel-details]][metabase.driver.sql-jdbc.connection.ssh-tunnel]- [metabase.driver.sqlite]))+ [metabase.driver.sqlite]+ [metabase.driver.util.settings]))diff --git a/src/metabase/driver/settings.clj b/src/metabase/driver/settings.cljindex d7141d3ae72..c727c87aed3 100644--- a/src/metabase/driver/settings.clj+++ b/src/metabase/driver/settings.clj@@ -261,15 +261,6 @@:type :integer:default 50000)-(defsetting engines- "Available database engines"- :encryption :no- :visibility :public- :setter :none- :getter (fn []- ((requiring-resolve 'metabase.driver.util/available-drivers-info)))- :doc false)-(defsetting sync-leaf-fields-limit(deferred-tru(str "Maximum number of leaf fields synced per collection of document database. Currently relevant for Mongo."diff --git a/src/metabase/driver/sql/query_processor.clj b/src/metabase/driver/sql/query_processor.cljindex 3a949fb82e3..fb9cbafb795 100644--- a/src/metabase/driver/sql/query_processor.clj+++ b/src/metabase/driver/sql/query_processor.clj@@ -12,6 +12,7 @@[metabase.driver-api.core :as driver-api][metabase.driver.common :as driver.common][metabase.driver.sql.query-processor.deprecated :as sql.qp.deprecated]+ [metabase.driver.sql.query-processor.format :as sql.qp.format][metabase.lib.core :as lib][metabase.lib.metadata.protocols :as lib.metadata.protocols][metabase.lib.options :as lib.options]@@ -26,6 +27,7 @@[metabase.util.malli :as mu][metabase.util.match :as match][metabase.util.performance :as perf :refer [empty? every? get-in mapv not-empty select-keys some]]+ [potemkin :as p][toucan2.pipeline :as t2.pipeline])(:import(java.time LocalDate LocalDateTime LocalTime OffsetDateTime OffsetTime ZonedDateTime)@@ -33,6 +35,11 @@(set! *warn-on-reflection* true)+(p/import-vars+ [sql.qp.format+ format-honeysql+ quote-style])+(def source-query-alias"Alias to use for source queries, e.g.:@@ -528,22 +535,6 @@(h2x/with-database-type-info (or (h2x/database-type day-of-week-honeysql-expr)"integer")))))))-(defmulti quote-style- "Return the dialect that should be used by Honey SQL 2 when building a SQL statement. Defaults to `:ansi`, but other- valid options are `:mysql`, `:sqlserver`, `:oracle`, and `:h2` (added in- [[metabase.util.honey-sql-2]]; like `:ansi`, but uppercases the result). Check [[honey.sql/dialects]] for all- available dialects, or register a custom one with [[honey.sql/register-dialect!]].-- (honey.sql/format ... :quoting (quote-style driver), :allow-dashed-names? true)-- (The name of this method reflects Honey SQL 1 terminology, where \"dialect\" was called \"quote style\". To avoid- needless churn, I haven't changed it yet. -- Cam)"- {:added "0.32.0" :arglists '([driver])}- driver/dispatch-on-initialized-driver- :hierarchy #'driver/hierarchy)--(defmethod quote-style :sql [_] :ansi)-(defmulti unix-timestamp->honeysql"Return a HoneySQL form appropriate for converting a Unix timestamp integer field or value to an proper SQL Timestamp.`seconds-or-milliseconds` refers to the resolution of the int in question and with be either `:seconds` or@@ -2065,39 +2056,6 @@(sort-by (fn [clause] [(get top-level-clause-application-order clause Integer/MAX_VALUE) clause])(keys inner-query)))-(defn- format-honeysql-2 [driver dialect honeysql-form]- ;; make sure [[driver/*driver*]] is bound, we need it for [[sqlize-value]]- (binding [driver/*driver* driver]- (sql/format honeysql-form {:dialect dialect- :quoted true- :quoted-snake false- :inline driver/*compile-with-inline-parameters*- ;; Enable :nested when we want to compile just one particular snippet.- :nested (not (map? honeysql-form))})))--(defmulti format-honeysql- "Compile `honeysql-form` to a `[sql & args]` vector. Prior to 0.51.0, this was a plain function, but was made a- multimethod in 0.51.0 to support drivers that need to always- specify [[metabase.driver/*compile-with-inline-parameters*]]."- {:arglists '([driver honeysql-form]), :added "0.51.0"}- driver/dispatch-on-initialized-driver- :hierarchy #'driver/hierarchy)--(defmethod format-honeysql :sql- [driver honeysql-form]- (let [dialect (quote-style driver)]- (try- (format-honeysql-2 driver dialect honeysql-form)- (catch Throwable e- (try- (log/error (u/format-color :red "Invalid HoneySQL form: %s" (ex-message e)))- (finally- (throw (ex-info (tru "Error compiling HoneySQL form: {0}" (ex-message e))- {:dialect dialect- :form honeysql-form- :type driver-api/qp.error-type.driver}- e))))))))-(defn- default-select [driver {[from] :from, :as _honeysql-form}](let [table-identifier (if (sequential? from);; Grab the alias part.diff --git a/src/metabase/driver/sql/query_processor/format.clj b/src/metabase/driver/sql/query_processor/format.cljnew file mode 100644index 00000000000..a541ee0f11a--- /dev/null+++ b/src/metabase/driver/sql/query_processor/format.clj@@ -0,0 +1,64 @@+(ns metabase.driver.sql.query-processor.format+ "Compiling a Honey SQL form to a SQL string for a driver.++ Drivers implement these as [[metabase.driver.sql.query-processor]] vars; that namespace re-exports them. They live+ here because [[metabase.driver.sql.util/quote-name]] needs [[format-honeysql]], and `sql.util` is in turn used+ by [[metabase.query-processor.util.persisted-cache]], which [[metabase.driver.sql.query-processor]] requires."+ (:require+ [honey.sql :as sql]+ [metabase.driver :as driver]+ [metabase.query-processor.error-type :as qp.error-type]+ [metabase.util :as u]+ [metabase.util.i18n :refer [tru]]+ [metabase.util.log :as log]))++(set! *warn-on-reflection* true)++(defmulti quote-style+ "Return the dialect that should be used by Honey SQL 2 when building a SQL statement. Defaults to `:ansi`, but other+ valid options are `:mysql`, `:sqlserver`, `:oracle`, and `:h2` (added in+ [[metabase.util.honey-sql-2]]; like `:ansi`, but uppercases the result). Check [[honey.sql/dialects]] for all+ available dialects, or register a custom one with [[honey.sql/register-dialect!]].++ (honey.sql/format ... :quoting (quote-style driver), :allow-dashed-names? true)++ (The name of this method reflects Honey SQL 1 terminology, where \"dialect\" was called \"quote style\". To avoid+ needless churn, I haven't changed it yet. -- Cam)"+ {:added "0.32.0" :arglists '([driver])}+ driver/dispatch-on-initialized-driver+ :hierarchy #'driver/hierarchy)++(defmethod quote-style :sql [_] :ansi)++(defn- format-honeysql-2 [driver dialect honeysql-form]+ ;; make sure [[driver/*driver*]] is bound, we need it for [[sqlize-value]]+ (binding [driver/*driver* driver]+ (sql/format honeysql-form {:dialect dialect+ :quoted true+ :quoted-snake false+ :inline driver/*compile-with-inline-parameters*+ ;; Enable :nested when we want to compile just one particular snippet.+ :nested (not (map? honeysql-form))})))++(defmulti format-honeysql+ "Compile `honeysql-form` to a `[sql & args]` vector. Prior to 0.51.0, this was a plain function, but was made a+ multimethod in 0.51.0 to support drivers that need to always+ specify [[metabase.driver/*compile-with-inline-parameters*]]."+ {:arglists '([driver honeysql-form]), :added "0.51.0"}+ driver/dispatch-on-initialized-driver+ :hierarchy #'driver/hierarchy)++(defmethod format-honeysql :sql+ [driver honeysql-form]+ (let [dialect (quote-style driver)]+ (try+ (format-honeysql-2 driver dialect honeysql-form)+ (catch Throwable e+ (try+ (log/error (u/format-color :red "Invalid HoneySQL form: %s" (ex-message e)))+ (finally+ (throw (ex-info (tru "Error compiling HoneySQL form: {0}" (ex-message e))+ {:dialect dialect+ :form honeysql-form+ :type qp.error-type/driver}+ e))))))))diff --git a/src/metabase/driver/sql/util.clj b/src/metabase/driver/sql/util.cljindex 3ff65c09f43..d0cb4711a94 100644--- a/src/metabase/driver/sql/util.clj+++ b/src/metabase/driver/sql/util.clj@@ -2,8 +2,8 @@"Utility functions for writing SQL drivers."(:require[clojure.string :as str]- [metabase.driver-api.core :as driver-api]- [metabase.driver.sql.query-processor :as sql.qp]+ [metabase.driver.sql.query-processor.format :as sql.qp.format]+ [metabase.query-processor.error-type :as qp.error-type][metabase.util :as u][metabase.util.honey-sql-2 :as h2x][metabase.util.i18n :refer [tru]]@@ -30,7 +30,7 @@identifier-type :- h2x/IdentifierType& components](first- (sql.qp/format-honeysql driver (apply h2x/identifier identifier-type components))))+ (sql.qp.format/format-honeysql driver (apply h2x/identifier identifier-type components))));;; +----------------------------------------------------------------------------------------------------------------+;;; | Deduplicate Field Aliases |@@ -205,12 +205,12 @@[has-timezone? target-timezone source-timezone](when (and has-timezone? source-timezone)(throw (ex-info (tru "input column already has a set timezone. Please remove the source parameter in convertTimezone.")- {:type driver-api/qp.error-type.invalid-query+ {:type qp.error-type/invalid-query:target-timezone target-timezone:source-timezone source-timezone})))(when (and (not has-timezone?) (not source-timezone))(throw (ex-info (tru "input column doesn''t have a set timezone. Please set the source parameter in convertTimezone to convert it.")- {:type driver-api/qp.error-type.invalid-query+ {:type qp.error-type/invalid-query:target-timezone target-timezone:source-timezone source-timezone}))))diff --git a/src/metabase/driver/sql_jdbc/connection.clj b/src/metabase/driver/sql_jdbc/connection.cljindex f41c9a18962..8dd15d9b522 100644--- a/src/metabase/driver/sql_jdbc/connection.clj+++ b/src/metabase/driver/sql_jdbc/connection.clj@@ -12,6 +12,7 @@[metabase.driver.db :as driver.db][metabase.driver.settings :as driver.settings][metabase.driver.sql-jdbc.common :as sql-jdbc.common]+ [metabase.driver.sql-jdbc.connection.pool-lock :as pool-lock][metabase.driver.sql-jdbc.connection.ssh-tunnel :as ssh][metabase.driver.util :as driver.u][metabase.util :as u]@@ -476,7 +477,7 @@;; We don't want to end up with a bunch of simultaneous threads creating pools only to have them destroyed;; the very next instant. This will cause their queries to fail. Thus we should do the usual locking here;; and make sure only one thread will be creating a pool at a given instant.- (locking pool-cache-key->connection-pool+ (locking pool-lock/monitor(or;; check if another thread created the pool while we were waiting to acquire the lock(get-canonical-pool cache-key details-hash false)diff --git a/src/metabase/driver/sql_jdbc/connection/pool_lock.clj b/src/metabase/driver/sql_jdbc/connection/pool_lock.cljnew file mode 100644index 00000000000..5d5d03ff258--- /dev/null+++ b/src/metabase/driver/sql_jdbc/connection/pool_lock.clj@@ -0,0 +1,12 @@+(ns metabase.driver.sql-jdbc.connection.pool-lock+ "The monitor serializing c3p0 pool creation against reads of c3p0's JMX attributes.++ Holding it on both sides is required to avoid a deadlock inside c3p0: reading a JMX attribute locks a+ DynamicPooledDataSourceManagerMBean and then a PoolBackedDataSource, while+ `com.mchange.v2.c3p0.DataSources/pooledDataSource` locks the two in the opposite order. See+ https://github.com/swaldman/c3p0/issues/95++ Carries no requires so that the JMX reader can take the same monitor without loading the driver.")++(defonce ^{:doc "The monitor object. Take it with `locking`."} monitor+ (Object.))diff --git a/src/metabase/driver/util/settings.clj b/src/metabase/driver/util/settings.cljnew file mode 100644index 00000000000..b6433e71363--- /dev/null+++ b/src/metabase/driver/util/settings.clj@@ -0,0 +1,20 @@+(ns metabase.driver.util.settings+ "The `engines` setting: which drivers this instance can use, and the connection form each one needs.++ It lives here rather than alongside the other driver settings because its value comes from+ [[metabase.driver.util/available-drivers-info]], while [[metabase.driver.util]] reads+ [[metabase.driver.settings]] -- defining it there would make those two namespaces require each other.++ Nothing refers to this namespace by name; the setting reaches the settings API only because+ [[metabase.driver.init]] loads it."+ (:require+ [metabase.driver.util :as driver.util]+ [metabase.settings.core :refer [defsetting]]))++(defsetting engines+ "Available database engines"+ :encryption :no+ :visibility :public+ :setter :none+ :getter driver.util/available-drivers-info+ :doc false)diff --git a/src/metabase/explorations/interestingness.clj b/src/metabase/explorations/interestingness.cljindex 0b6d6b2d2a6..36db4767b52 100644--- a/src/metabase/explorations/interestingness.clj+++ b/src/metabase/explorations/interestingness.clj@@ -45,7 +45,7 @@(extraction-units (lib/raw-temporal-bucket lib-col)))(defn- start-of-week-day ^DayOfWeek []- (-> ((requiring-resolve 'metabase.settings.core/get) :start-of-week)+ (-> (lib-be/start-of-week)(or :sunday)name(u/upper-case-en)diff --git a/src/metabase/indexed_entities/models/model_index.clj b/src/metabase/indexed_entities/models/model_index.cljindex 8d669dc58b9..5f63f94085c 100644--- a/src/metabase/indexed_entities/models/model_index.clj+++ b/src/metabase/indexed_entities/models/model_index.clj@@ -2,6 +2,7 @@(:require[clojure.set :as set][clojure.string :as str]+ [clojurewerkz.quartzite.triggers :as triggers][metabase.indexed-entities.db :as indexed-entities.db];; legacy usage, do not use this in new code^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.normalize :as mbql.normalize]@@ -14,6 +15,7 @@[metabase.query-processor.core :as qp][metabase.search.core :as search][metabase.sync.schedules :as sync.schedules]+ [metabase.task.core :as task][metabase.util.cron :as u.cron][metabase.util.log :as log][metabase.util.malli :as mu]@@ -39,10 +41,22 @@{:pk_ref mi/transform-legacy-field-ref:value_ref mi/transform-legacy-field-ref})+(defn trigger-key+ "Quartz trigger key for the job that refreshes the values of model index `model-index-id`."+ [model-index-id]+ (triggers/key (format "metabase.task.IndexValues.trigger.%d" model-index-id)))++(defn remove-indexing-job+ "Remove the indexing job for `model-index`.++ This and [[trigger-key]] live here rather than with the job itself so that deleting a ModelIndex can cancel its+ trigger without reaching into [[metabase.indexed-entities.task.index-values]], which reads this namespace."+ [model-index]+ (task/delete-trigger! (trigger-key (:id model-index))))+(t2/define-before-delete :model/ModelIndex[model-index]- (let [remove-refresh-job (requiring-resolve 'metabase.indexed-entities.task.index-values/remove-indexing-job)]- (remove-refresh-job model-index)))+ (remove-indexing-job model-index))(def max-indexed-values"Maximum number of values we will index. Actually take one more than this to test if there are more than thediff --git a/src/metabase/indexed_entities/task/index_values.clj b/src/metabase/indexed_entities/task/index_values.cljindex 9bc421c5645..e681a28e788 100644--- a/src/metabase/indexed_entities/task/index_values.clj+++ b/src/metabase/indexed_entities/task/index_values.clj@@ -32,11 +32,6 @@(not= (:type model) :model)(:archived model)))-(defn- model-index-trigger-key- [model-index-id]- (triggers/key- (format "metabase.task.IndexValues.trigger.%d" model-index-id)))-(defn- refresh-index!"Refresh the index on a model. Note, if the index should be removed (no longer a model, archived,etc, (see [[should-deindex?]])) will delete the indexing job."@@ -46,7 +41,7 @@(indexed-entities.db/card (:model_id model-index)))](if (should-deindex? model model-index)(u/ignore-exceptions- (let [trigger-key (model-index-trigger-key model-index-id)]+ (let [trigger-key (model-index/trigger-key model-index-id)](task/delete-trigger! trigger-key)(indexed-entities.db/delete-model-index! model-index-id)))(model-index/add-values! model-index))))@@ -72,7 +67,7 @@(defn- refresh-trigger ^org.quartz.CronTrigger [model-index](triggers/build(triggers/with-description (format "Refresh index on model %d" (:model_id model-index)))- (triggers/with-identity (model-index-trigger-key (:id model-index)))+ (triggers/with-identity (model-index/trigger-key (:id model-index)))(triggers/using-job-data {"model-index-id" (u/the-id model-index)})(triggers/for-job (jobs/key refresh-model-index-key))(triggers/start-now)@@ -95,12 +90,6 @@(catch Exception e(log/warnf "Error scheduling indexing for model: %s: %s" (:model_id model-index) (ex-message e))))))-(defn remove-indexing-job- "Public API to remove an indexing job on a model."- [model-index]- (let [trigger-key (model-index-trigger-key (:id model-index))]- (task/delete-trigger! trigger-key)))-(defn- recreate-missing-triggers!"Ensure all model indexes in the database have triggers in Quartz."[]diff --git a/src/metabase/mcp/v2/projections.clj b/src/metabase/mcp/v2/projections.cljindex 87957b86ec0..47ef4f095cd 100644--- a/src/metabase/mcp/v2/projections.clj+++ b/src/metabase/mcp/v2/projections.clj@@ -16,7 +16,7 @@(:require[clojure.string :as str][medley.core :as m]- [metabase.dashboards.models.dashboard :as dashboard]+ [metabase.parameters.params :as params][metabase.util :as u]))(set! *warn-on-reflection* true)@@ -252,9 +252,9 @@(defn- dashboard-parameters-summary"One row per dashboard parameter: id, name, type, and the ids of the dashcards it is wired- to (via [[dashboard/dashboard->resolved-params]], the same resolution the QP uses)."+ to (via [[params/dashboard->resolved-params]], the same resolution the QP uses)."[dash]- (let [resolved (dashboard/dashboard->resolved-params dash)]+ (let [resolved (params/dashboard->resolved-params dash)](mapv (fn [{param-id :id :as param}](m/remove-valsnil?diff --git a/src/metabase/models/interface.clj b/src/metabase/models/interface.cljindex 5377db1f282..29f7f062e63 100644--- a/src/metabase/models/interface.clj+++ b/src/metabase/models/interface.clj@@ -15,6 +15,7 @@[clojure.string :as str][clojure.walk :as walk][medley.core :as m]+ [metabase.app-db.core :as mdb];; Toucan out-transforms normalize stored legacy MBQL on read; needed until the app db is MBQL 5^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.normalize :as mbql.normalize];; stored card queries/refs are still legacy MBQL; validated against the legacy schema on read/write@@ -552,7 +553,7 @@and H2 and `now(6)` for MySQL/MariaDB (`now()` for MySQL only return second resolution; `now(6)` uses themax (nanosecond) resolution)."[]- (h2x/current-datetime-honeysql-form ((requiring-resolve 'metabase.app-db.core/db-type))))+ (h2x/current-datetime-honeysql-form (mdb/db-type)))(defn- add-created-at-timestamp [obj & _](cond-> objdiff --git a/src/metabase/models/serialization.clj b/src/metabase/models/serialization.cljindex c1353901a60..7dae9ebdd32 100644--- a/src/metabase/models/serialization.clj+++ b/src/metabase/models/serialization.clj@@ -70,7 +70,9 @@[metabase.lib.schema.parameter :as lib.schema.parameter][metabase.models.db :as models.db][metabase.models.interface :as mi]+ [metabase.models.serialization.path][metabase.models.serialization.resolve :as resolve]+ [metabase.models.serialization.resolve.default :as resolve.default][metabase.models.visualization-settings :as mb.viz][metabase.util :as u][metabase.util.date-2 :as u.date]@@ -80,12 +82,22 @@[metabase.util.malli.humanize :as mu.humanize][metabase.util.malli.registry :as mr][metabase.util.match :as match]+ [potemkin :as p][toucan2.core :as t2][toucan2.model :as t2.model][toucan2.realize :as t2.realize]))(set! *warn-on-reflection* true)+(p/import-vars+ [metabase.models.serialization.path+ entity-id+ field-hierarchy+ generate-path+ infer-self-path+ lookup-by-id+ maybe-labeled])+;; there was no science behind picking 100 as a number(def ^:private extract-nested-batch-limit "max amount of entities to fetch nested entities for" 100)@@ -93,17 +105,6 @@"Maximum number of ids per `:in` clause, to stay under database parameter limits."1000)-(mr/def ::model-keyword- [:and- qualified-keyword?- [:fn- {:error/message ":model/X keyword"}- #(when (qualified-keyword? %)- (= (namespace %) "model"))]])--(mr/def ::model-keyword-or-symbol- [:or symbol? ::model-keyword])-;;; # Serialization Overview;;;;;; Serialization (or "serdes") is a system for exporting entities (Dashboards, Cards, Collections, etc.) from one@@ -142,19 +143,6 @@;;; - For entities that existed before the column was added, have a portable way to rebuild them (see below on;;; hashing).-(defmulti entity-id- "Given the model name and an entity, returns its entity ID (which might be nil).-- This abstracts over the exact definition of the \"entity ID\" for a given entity.- By default this is a column, `:entity_id`.-- Models that have a different portable ID (`Database`, `Field`, etc.) should override this."- {:arglists '([model-name instance])}- (fn [model-name _instance] model-name))--(defmethod entity-id :default [_ instance]- (some-> instance :entity_id str/trim))-(defn has-entity-id?"Returns true if the model has an `:entity_id` column."[model]@@ -200,41 +188,6 @@;;; ## Two kinds of nesting;;; To reiterate, `:serdes/meta` paths are not filesystem paths. When `extract`ed entities are stored to disk.-(defmulti generate-path- "Given the model name and raw entity from the database, returns a vector giving its *path*.- `(generate-path \"ModelName\" entity)`-- The path is a vector of maps, root first and this entity itself last. Each map looks like:- `{:model \"ModelName\" :id \"entity ID, identity hash, or custom ID\" :label \"optional human label\"}`-- Nested models with no entity_id need to return nil for generate-path."- {:arglists '([model-name instance])}- (fn [model-name _instance] model-name))--(defn infer-self-path- "Returns `{:model \"ModelName\" :id \"id-string\"}`"- [model-name entity]- {:model model-name- :id (entity-id model-name entity)})--(defn maybe-labeled- "Common helper for defining [[generate-path]] for an entity that is- (1) top-level, ie. a one layer path;- (2) labeled by a single field, slugified.-- For example, a Card's or Dashboard's `:name` field."- [model-name entity slug-key]- (let [self (infer-self-path model-name entity)- label (slug-key entity)]- [(-> self- (m/assoc-some :label (some-> label (u/slugify {:unicode? true}))))]))--(defmethod generate-path :default [model-name entity]- ;; This default works for most models, but needs overriding for those that don't rely on entity_id.- (maybe-labeled model-name entity #(if (string? (:name %))- (:name %)- (:format-string (:name %)))))-(defn log-path-str"Returns a string for logging from a serdes path sequence (i.e. in :serdes/meta)"[elements]@@ -643,8 +596,6 @@(fn [path](-> path last :model)))-(declare lookup-by-id)-(defmethod load-find-local :default [path](let [{id :id model-name :model} (last path)model (t2.model/resolve-model (symbol model-name))]@@ -799,12 +750,6 @@[id-str](resolve/entity-id? id-str))-(mu/defn lookup-by-id- "Given an entity ID string, finds the matching entity. This is useful when writing [[xform-one]] to- turn a foreign key from a portable form to an appdb ID. Returns a Toucan entity or nil."- [model :- ::model-keyword-or-symbol id-str]- (models.db/entity-by-entity-id model id-str))-(defn storage-default-collection-path"Implements the most common structure for [[storage-path]].Returns a vector of maps with `:label` and `:key` for each path segment.@@ -861,17 +806,13 @@;;; These wrapper functions delegate to the current resolver (set by [[with-cache]]).;;; When no resolver is bound, they fall back to the database-backed resolver.-;; TODO: `requiring-resolve` is needed here because resolve.default requires this ns-;; (for `generate-path`, `field-hierarchy`, `lookup-by-id`).-;; Moving those into resolve.default (or a shared utils ns) would break the cycle and-;; let us require resolve.default directly.(defn- export-resolver [](or resolve/*export-resolver*- @(requiring-resolve 'metabase.models.serialization.resolve.default/default-export-resolver)))+ resolve.default/default-export-resolver))(defn- import-resolver [](or resolve/*import-resolver*- @(requiring-resolve 'metabase.models.serialization.resolve.default/default-import-resolver)))+ resolve.default/default-import-resolver));;; ## General foreign keys@@ -883,7 +824,7 @@NOTE: This works for both top-level and nested entities. Top-level entities like `Card` are returned as just aportable ID string.. Nested entities are returned as a vector of such ID strings."[id :- [:maybe pos-int?]- model :- ::model-keyword-or-symbol]+ model :- :metabase.models.serialization.path/model-keyword-or-symbol](resolve/export-fk (export-resolver) id model))(defmacro ^:private fk-elide@@ -909,7 +850,7 @@Unusual parameter order means this can be used as `(update x :some_id import-fk 'SomeModel)`."[eid- model :- ::model-keyword-or-symbol]+ model :- :metabase.models.serialization.path/model-keyword-or-symbol](resolve/import-fk (import-resolver) eid model))(mu/defn ^:dynamic *export-fk-keyed*@@ -920,7 +861,7 @@Note: This assumes the primary key is called `:id`."[id- model :- ::model-keyword-or-symbol+ model :- :metabase.models.serialization.path/model-keyword-or-symbolfield](resolve/export-fk-keyed (export-resolver) id model field))@@ -1019,12 +960,6 @@;;; ## Fields-(defn field-hierarchy- "Returns the field hierarchy (field + parents) for a field ID. Used by resolvers."- [id]- (reverse- (models.db/field-hierarchy-rows id)))-;; NOTE: field lookups are intentionally NOT routed through the cached resolver, unlike the;; database and table exporters above. Fields are unbounded in number (millions on large;; instances), and the dominant traffic — each field's own path during a data-model export —@@ -1784,7 +1719,7 @@left in its portable entity-id form, instead of throwing. Use for paths where a deleted source Cardshould be treated as a broken section rather than break the whole read."[settings]- (binding [resolve/*import-resolver* @(requiring-resolve 'metabase.models.serialization.resolve.default/lenient-import-resolver)]+ (binding [resolve/*import-resolver* resolve.default/lenient-import-resolver](import-visualizer-settings settings)))(defn import-visualization-settings@@ -1980,6 +1915,6 @@(defmacro with-cache"Runs body with resolvers bound to cached (memoized) versions for performance."[& body]- `(binding [resolve/*export-resolver* ((requiring-resolve 'metabase.models.serialization.resolve.default/cached-export-resolver))- resolve/*import-resolver* ((requiring-resolve 'metabase.models.serialization.resolve.default/cached-import-resolver))]+ `(binding [resolve/*export-resolver* (resolve.default/cached-export-resolver)+ resolve/*import-resolver* (resolve.default/cached-import-resolver)]~@body))diff --git a/src/metabase/models/serialization/path.clj b/src/metabase/models/serialization/path.cljnew file mode 100644index 00000000000..785df762d75--- /dev/null+++ b/src/metabase/models/serialization/path.clj@@ -0,0 +1,86 @@+(ns metabase.models.serialization.path+ "Portable identity for serdes entities: entity IDs, the `:serdes/meta` path built from them, and the appdb lookups+ that go the other way.++ Lives below [[metabase.models.serialization]] so that resolver implementations can use it without depending on the+ serdes interface, which depends on them in turn."+ (:require+ [clojure.string :as str]+ [medley.core :as m]+ [metabase.models.db :as models.db]+ [metabase.util :as u]+ [metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr]))++(set! *warn-on-reflection* true)++(mr/def ::model-keyword+ [:and+ qualified-keyword?+ [:fn+ {:error/message ":model/X keyword"}+ #(when (qualified-keyword? %)+ (= (namespace %) "model"))]])++(mr/def ::model-keyword-or-symbol+ [:or symbol? ::model-keyword])++(defmulti entity-id+ "Given the model name and an entity, returns its entity ID (which might be nil).++ This abstracts over the exact definition of the \"entity ID\" for a given entity.+ By default this is a column, `:entity_id`.++ Models that have a different portable ID (`Database`, `Field`, etc.) should override this."+ {:arglists '([model-name instance])}+ (fn [model-name _instance] model-name))++(defmethod entity-id :default [_ instance]+ (some-> instance :entity_id str/trim))++(defmulti generate-path+ "Given the model name and raw entity from the database, returns a vector giving its *path*.+ `(generate-path \"ModelName\" entity)`++ The path is a vector of maps, root first and this entity itself last. Each map looks like:+ `{:model \"ModelName\" :id \"entity ID, identity hash, or custom ID\" :label \"optional human label\"}`++ Nested models with no entity_id need to return nil for generate-path."+ {:arglists '([model-name instance])}+ (fn [model-name _instance] model-name))++(defn infer-self-path+ "Returns `{:model \"ModelName\" :id \"id-string\"}`"+ [model-name entity]+ {:model model-name+ :id (entity-id model-name entity)})++(defn maybe-labeled+ "Common helper for defining [[generate-path]] for an entity that is+ (1) top-level, ie. a one layer path;+ (2) labeled by a single field, slugified.++ For example, a Card's or Dashboard's `:name` field."+ [model-name entity slug-key]+ (let [self (infer-self-path model-name entity)+ label (slug-key entity)]+ [(-> self+ (m/assoc-some :label (some-> label (u/slugify {:unicode? true}))))]))++(defmethod generate-path :default [model-name entity]+ ;; This default works for most models, but needs overriding for those that don't rely on entity_id.+ (maybe-labeled model-name entity #(if (string? (:name %))+ (:name %)+ (:format-string (:name %)))))++(mu/defn lookup-by-id+ "Given an entity ID string, finds the matching entity. This is useful when writing [[metabase.models.serialization/xform-one]] to+ turn a foreign key from a portable form to an appdb ID. Returns a Toucan entity or nil."+ [model :- ::model-keyword-or-symbol id-str]+ (models.db/entity-by-entity-id model id-str))++(defn field-hierarchy+ "Returns the field hierarchy (field + parents) for a field ID. Used by resolvers."+ [id]+ (reverse+ (models.db/field-hierarchy-rows id)))diff --git a/src/metabase/models/serialization/resolve/default.clj b/src/metabase/models/serialization/resolve/default.cljindex 7324924fe0c..86960b40944 100644--- a/src/metabase/models/serialization/resolve/default.clj+++ b/src/metabase/models/serialization/resolve/default.clj@@ -5,7 +5,7 @@against the application database."(:require[metabase.models.db :as models.db]- [metabase.models.serialization :as serdes]+ [metabase.models.serialization.path :as serdes.path][metabase.models.serialization.resolve :as resolve][metabase.util.log :as log][toucan2.core :as t2])@@ -24,7 +24,7 @@(let [model-name (name model)entity (models.db/entity-by-own-pk model id)path (when entity- (mapv :id (serdes/generate-path model-name entity)))]+ (mapv :id (serdes.path/generate-path model-name entity)))](cond(nil? entity) (throw (ex-info "FK target not found" {:model model:id id@@ -55,7 +55,7 @@"Given a numeric field_id, return a portable field reference [db-name schema table-name field-name]."[resolver field-id](when field-id- (let [fields (serdes/field-hierarchy field-id)+ (let [fields (serdes.path/field-hierarchy field-id)[db-name schema field-name] (resolve/export-table-fk resolver (:table_id (first fields)))](into [db-name schema field-name] (map :name fields)))))@@ -68,7 +68,7 @@[eid model](when eid(let [eid (if (vector? eid) (last eid) eid)- entity (serdes/lookup-by-id model eid)]+ entity (serdes.path/lookup-by-id model eid)](if entity(get entity (first (t2/primary-keys model)))(throw (ex-info "Could not find foreign key target - bad serdes dependencies or other serialization error"diff --git a/src/metabase/notification/events/report_timezone_updated.clj b/src/metabase/notification/events/report_timezone_updated.cljindex 07baa71998c..60e4ce35a5f 100644--- a/src/metabase/notification/events/report_timezone_updated.clj+++ b/src/metabase/notification/events/report_timezone_updated.clj@@ -1,7 +1,7 @@(ns metabase.notification.events.report-timezone-updated(:require[metabase.events.core :as events]- [metabase.notification.task.send :as send]+ [metabase.notification.task.send-trigger :as notification.task.send-trigger][methodical.core :as methodical]))(events/derive! ::event :metabase/event)@@ -10,4 +10,4 @@(methodical/defmethod events/publish-event! ::event"When the report-timezone Setting is updated, update the timezone of all SendNotification triggers."[_topic _event]- (send/update-send-notification-triggers-timezone!))+ (notification.task.send-trigger/update-send-notification-triggers-timezone!))diff --git a/src/metabase/notification/models.clj b/src/metabase/notification/models.cljindex f314fe90c52..3c0c7547dcf 100644--- a/src/metabase/notification/models.clj+++ b/src/metabase/notification/models.clj@@ -11,6 +11,7 @@[metabase.models.interface :as mi][metabase.models.util.spec-update :as models.u.spec-update][metabase.notification.db :as notification.db]+ [metabase.notification.task.send-trigger :as notification.task.send-trigger][metabase.permissions.core :as perms][metabase.permissions.schema :as permissions.schema][metabase.premium-features.core :as premium-features :refer [defenterprise]]@@ -148,14 +149,6 @@(validate-notification instance)instance)-(defn- update-subscription-trigger!- [& args]- (apply (requiring-resolve 'metabase.notification.task.send/update-subscription-trigger!) args))--(defn- delete-trigger-for-subscription!- [& args]- (apply (requiring-resolve 'metabase.notification.task.send/delete-trigger-for-subscription!) args))-(t2/define-before-update :model/Notification[instance](validate-notification instance)@@ -175,14 +168,14 @@(let [subscriptions (notification.db/cron-subscriptions-for-notification (:id instance))](doseq [subscription subscriptions](if (:active instance)- (update-subscription-trigger! subscription)- (delete-trigger-for-subscription! (:id subscription))))))+ (notification.task.send-trigger/update-subscription-trigger! subscription)+ (notification.task.send-trigger/delete-trigger-for-subscription! (:id subscription))))))instance)(t2/define-before-delete :model/Notification[instance](doseq [subscription-id (notification.db/cron-subscription-ids-for-notification (:id instance))]- (delete-trigger-for-subscription! subscription-id))+ (notification.task.send-trigger/delete-trigger-for-subscription! subscription-id))(when-let [payload-id (:payload_id instance)](case (:payload_type instance):notification/card (notification.db/delete-notification-card! payload-id)))@@ -248,18 +241,18 @@(t2/define-after-insert :model/NotificationSubscription[instance]- (update-subscription-trigger! instance)+ (notification.task.send-trigger/update-subscription-trigger! instance)instance)(t2/define-before-update :model/NotificationSubscription[instance](validate-subscription instance)- (update-subscription-trigger! instance)+ (notification.task.send-trigger/update-subscription-trigger! instance)instance)(t2/define-before-delete :model/NotificationSubscription[instance]- (delete-trigger-for-subscription! (:id instance))+ (notification.task.send-trigger/delete-trigger-for-subscription! (:id instance))instance);; ------------------------------------------------------------------------------------------------;;diff --git a/src/metabase/notification/task/send.clj b/src/metabase/notification/task/send.cljindex ef7caf94908..1ed0c0e73ea 100644--- a/src/metabase/notification/task/send.clj+++ b/src/metabase/notification/task/send.clj@@ -1,103 +1,20 @@(ns metabase.notification.task.send(:require- [clojure.data :refer [diff]][clojurewerkz.quartzite.conversion :as qc][clojurewerkz.quartzite.jobs :as jobs]- [clojurewerkz.quartzite.schedule.cron :as cron][clojurewerkz.quartzite.triggers :as triggers]- [metabase.driver :as driver][metabase.notification.db :as notification.db][metabase.notification.send :as notification.send]- [metabase.query-processor.timezone :as qp.timezone]+ [metabase.notification.task.send-trigger :as notification.task.send-trigger][metabase.task-history.core :as task-history][metabase.task.core :as task][metabase.tracing.core :as tracing][metabase.util.log :as log])(:import- (java.util TimeZone)- (org.quartz CronTrigger DisallowConcurrentExecution JobExecutionContext TriggerKey)))+ (org.quartz DisallowConcurrentExecution JobExecutionContext)))(set! *warn-on-reflection* true)-(def ^:private send-notification-job-key (jobs/key "metabase.task.notification.send.job"))--(defn- send-notification-timezone- []- (or (driver/report-timezone)- (qp.timezone/system-timezone-id)- "UTC"))--(defn- send-notification-trigger-key- ^TriggerKey [subscription-id]- (triggers/key (format "metabase.task.notification.trigger.subscription.%d"- subscription-id)))--(defn- send-notification-trigger-key->subscription-id- [trigger-key]- (when-let [[_ m] (re-matches #"metabase\.task\.notification\.trigger\.subscription\.(\d+)" (str trigger-key))]- (parse-long m)))--(defn- build-trigger- "Build a Quartz Trigger for `database` and `task-info` if a schedule exists."- ^CronTrigger [subscription-id cron-schedule]- (triggers/build- (triggers/with-description (format "Notification Subscription %d" subscription-id))- (triggers/with-identity (send-notification-trigger-key subscription-id))- (triggers/using-job-data {"subscription-id" subscription-id})- (triggers/for-job send-notification-job-key)- (triggers/start-now)- (triggers/with-schedule- (cron/schedule- (cron/cron-schedule cron-schedule)- (cron/in-time-zone (TimeZone/getTimeZone ^String (send-notification-timezone)))- ;; We want to fire the trigger once even if the previous triggers missed- (cron/with-misfire-handling-instruction-fire-and-proceed)))- ;; higher than sync- (triggers/with-priority 6)))--(defn- create-new-trigger!- [{:keys [id cron_schedule] :as _notification-subscription}]- (task/add-trigger! (build-trigger id cron_schedule)))--(defn update-subscription-trigger!- "Update the trigger for a notification subscription if it exists and needs to be updated."- [{:keys [id type cron_schedule] :as notification-subscription}]- (let [existing-trigger (first (task/existing-triggers send-notification-job-key (send-notification-trigger-key id)))]- (cond- ;; delete trigger if type changes- (and- (not= type :notification-subscription/cron)- existing-trigger)- (do- (log/infof "Deleting trigger for subscription %d because of type changes" id)- (task/delete-trigger! (-> existing-trigger :key triggers/key)))-- ;; do nothing if type is not cron- (not= type :notification-subscription/cron)- nil-- ;; create new if there is no existing trigger- (not existing-trigger)- (do- (log/infof "Creating new trigger for subscription %d with schedule %s" id cron_schedule)- (create-new-trigger! notification-subscription))-- (not= cron_schedule (:schedule existing-trigger))- (do- (log/infof "Rescheduling trigger for subscription %d from %s to %s" id (:schedule existing-trigger) cron_schedule)- (task/delete-trigger! (-> existing-trigger :key triggers/key))- (create-new-trigger! notification-subscription))-- :else- (log/infof "No changes to trigger for subscription %d" id))))--(defn delete-trigger-for-subscription!- "Delete the trigger for a notification subscription."- [notification-subscription-id]- (when-first [trigger (task/existing-triggers send-notification-job-key (send-notification-trigger-key notification-subscription-id))]- (log/infof "Deleting trigger for subscription %d" notification-subscription-id)- (task/delete-trigger! (-> trigger :key triggers/key))))-(defn- send-notification*[subscription-id](let [subscription (notification.db/subscription subscription-id)@@ -126,29 +43,12 @@(nil? notification)(do(log/warnf "Skipping and deleting trigger for subscription %d because it does not exist." subscription-id)- (delete-trigger-for-subscription! subscription-id))+ (notification.task.send-trigger/delete-trigger-for-subscription! subscription-id))(not (:active notification))(do(log/warnf "Skipping and deleting trigger for subscription %d because the notification is deactivated" subscription-id)- (delete-trigger-for-subscription! subscription-id))))))--(defn- active-cron-subscription-id->subscription- []- (notification.db/active-cron-subscriptions-by-id))--(defn update-send-notification-triggers-timezone!- "Update the timezone of all SendNotification triggers if the report timezone changes."- []- (let [triggers (-> send-notification-job-key task/job-info :triggers)- new-timezone (send-notification-timezone)- subscription-id->cron (update-vals (active-cron-subscription-id->subscription) :cron_schedule)]- (doseq [trigger triggers- :when (not= new-timezone (:timezone trigger))] ; skip if timezone is the same- (let [trigger-key (:key trigger)- subscription-id (send-notification-trigger-key->subscription-id trigger-key)]- (log/infof "Updating timezone of trigger %s to %s. Was: %s" trigger-key new-timezone (:timezone trigger))- (task/reschedule-trigger! (build-trigger subscription-id (get subscription-id->cron subscription-id)))))))+ (notification.task.send-trigger/delete-trigger-for-subscription! subscription-id))))))(task/defjob ^{:doc "Triggers that send a notification for a subscription."}SendNotification@@ -173,24 +73,6 @@subscription-id trigger-key scheduled-fire-time fire-time recovering? refire-count scheduler-id)(send-notification* subscription-id))))-(defn init-send-notification-triggers!- "Initialize all notification subscription triggers.-- Called when starting the instance."- []- (assert (task/scheduler) "Scheduler must be started before initializing SendNotification triggers")-- ;; Get all existing triggers and subscription IDs- (let [existing-triggers (:triggers (task/job-info send-notification-job-key))- existing-triggers-subscription-ids (map #(get-in % [:data "subscription-id"]) existing-triggers)- subscription-id->subscription (active-cron-subscription-id->subscription)- db-subscription-ids (keys subscription-id->subscription)- [to-delete to-create _to-skip] (diff (set existing-triggers-subscription-ids) (set db-subscription-ids))]- (doseq [subscription-id to-delete]- (delete-trigger-for-subscription! subscription-id))- (doseq [subscription-id to-create]- (create-new-trigger! (get subscription-id->subscription subscription-id)))))-(task/defjob^{:doc"Find all notification subscriptions with cron schedules and create a trigger for each.@@ -204,11 +86,11 @@InitNotificationTriggers[_context](log/info "Initializing SendNotification triggers")- (init-send-notification-triggers!))+ (notification.task.send-trigger/init-send-notification-triggers!))(defmethod task/init! ::SendNotifications [_](let [send-notification-job (jobs/build- (jobs/with-identity send-notification-job-key)+ (jobs/with-identity notification.task.send-trigger/send-notification-job-key)(jobs/with-description "Send Notification")(jobs/of-type SendNotification)(jobs/store-durably))diff --git a/src/metabase/notification/task/send_trigger.clj b/src/metabase/notification/task/send_trigger.cljnew file mode 100644index 00000000000..88101cdc4e7--- /dev/null+++ b/src/metabase/notification/task/send_trigger.clj@@ -0,0 +1,136 @@+(ns metabase.notification.task.send-trigger+ "Quartz trigger lifecycle for notification subscriptions.++ Separate from [[metabase.notification.task.send]] so that Notification model hooks can (un)schedule triggers: the job+ body over there loads the sending machinery, which reads the models."+ (:require+ [clojure.data :refer [diff]]+ [clojurewerkz.quartzite.jobs :as jobs]+ [clojurewerkz.quartzite.schedule.cron :as cron]+ [clojurewerkz.quartzite.triggers :as triggers]+ [metabase.driver :as driver]+ [metabase.notification.db :as notification.db]+ [metabase.query-processor.timezone :as qp.timezone]+ [metabase.task.core :as task]+ [metabase.util.log :as log])+ (:import+ (java.util TimeZone)+ (org.quartz CronTrigger TriggerKey)))++(set! *warn-on-reflection* true)++(def send-notification-job-key+ "Key of the Quartz job that sends a notification when a subscription's trigger fires."+ (jobs/key "metabase.task.notification.send.job"))++(defn- send-notification-timezone+ []+ (or (driver/report-timezone)+ (qp.timezone/system-timezone-id)+ "UTC"))++(defn- send-notification-trigger-key+ ^TriggerKey [subscription-id]+ (triggers/key (format "metabase.task.notification.trigger.subscription.%d"+ subscription-id)))++(defn- send-notification-trigger-key->subscription-id+ [trigger-key]+ (when-let [[_ m] (re-matches #"metabase\.task\.notification\.trigger\.subscription\.(\d+)" (str trigger-key))]+ (parse-long m)))++(defn- build-trigger+ "Build a Quartz Trigger for `database` and `task-info` if a schedule exists."+ ^CronTrigger [subscription-id cron-schedule]+ (triggers/build+ (triggers/with-description (format "Notification Subscription %d" subscription-id))+ (triggers/with-identity (send-notification-trigger-key subscription-id))+ (triggers/using-job-data {"subscription-id" subscription-id})+ (triggers/for-job send-notification-job-key)+ (triggers/start-now)+ (triggers/with-schedule+ (cron/schedule+ (cron/cron-schedule cron-schedule)+ (cron/in-time-zone (TimeZone/getTimeZone ^String (send-notification-timezone)))+ ;; We want to fire the trigger once even if the previous triggers missed+ (cron/with-misfire-handling-instruction-fire-and-proceed)))+ ;; higher than sync+ (triggers/with-priority 6)))++(defn- create-new-trigger!+ [{:keys [id cron_schedule] :as _notification-subscription}]+ (task/add-trigger! (build-trigger id cron_schedule)))++(defn update-subscription-trigger!+ "Update the trigger for a notification subscription if it exists and needs to be updated."+ [{:keys [id type cron_schedule] :as notification-subscription}]+ (let [existing-trigger (first (task/existing-triggers send-notification-job-key (send-notification-trigger-key id)))]+ (cond+ ;; delete trigger if type changes+ (and+ (not= type :notification-subscription/cron)+ existing-trigger)+ (do+ (log/infof "Deleting trigger for subscription %d because of type changes" id)+ (task/delete-trigger! (-> existing-trigger :key triggers/key)))++ ;; do nothing if type is not cron+ (not= type :notification-subscription/cron)+ nil++ ;; create new if there is no existing trigger+ (not existing-trigger)+ (do+ (log/infof "Creating new trigger for subscription %d with schedule %s" id cron_schedule)+ (create-new-trigger! notification-subscription))++ (not= cron_schedule (:schedule existing-trigger))+ (do+ (log/infof "Rescheduling trigger for subscription %d from %s to %s" id (:schedule existing-trigger) cron_schedule)+ (task/delete-trigger! (-> existing-trigger :key triggers/key))+ (create-new-trigger! notification-subscription))++ :else+ (log/infof "No changes to trigger for subscription %d" id))))++(defn delete-trigger-for-subscription!+ "Delete the trigger for a notification subscription."+ [notification-subscription-id]+ (when-first [trigger (task/existing-triggers send-notification-job-key (send-notification-trigger-key notification-subscription-id))]+ (log/infof "Deleting trigger for subscription %d" notification-subscription-id)+ (task/delete-trigger! (-> trigger :key triggers/key))))++(defn- active-cron-subscription-id->subscription+ []+ (notification.db/active-cron-subscriptions-by-id))++(defn update-send-notification-triggers-timezone!+ "Update the timezone of all SendNotification triggers if the report timezone changes."+ []+ (let [triggers (-> send-notification-job-key task/job-info :triggers)+ new-timezone (send-notification-timezone)+ subscription-id->cron (update-vals (active-cron-subscription-id->subscription) :cron_schedule)]+ (doseq [trigger triggers+ :when (not= new-timezone (:timezone trigger))] ; skip if timezone is the same+ (let [trigger-key (:key trigger)+ subscription-id (send-notification-trigger-key->subscription-id trigger-key)]+ (log/infof "Updating timezone of trigger %s to %s. Was: %s" trigger-key new-timezone (:timezone trigger))+ (task/reschedule-trigger! (build-trigger subscription-id (get subscription-id->cron subscription-id)))))))++(defn init-send-notification-triggers!+ "Initialize all notification subscription triggers.++ Called when starting the instance."+ []+ (assert (task/scheduler) "Scheduler must be started before initializing SendNotification triggers")++ ;; Get all existing triggers and subscription IDs+ (let [existing-triggers (:triggers (task/job-info send-notification-job-key))+ existing-triggers-subscription-ids (map #(get-in % [:data "subscription-id"]) existing-triggers)+ subscription-id->subscription (active-cron-subscription-id->subscription)+ db-subscription-ids (keys subscription-id->subscription)+ [to-delete to-create _to-skip] (diff (set existing-triggers-subscription-ids) (set db-subscription-ids))]+ (doseq [subscription-id to-delete]+ (delete-trigger-for-subscription! subscription-id))+ (doseq [subscription-id to-create]+ (create-new-trigger! (get subscription-id->subscription subscription-id)))))diff --git a/src/metabase/parameters/params.clj b/src/metabase/parameters/params.cljindex 04ba37baab0..20f67bcbeec 100644--- a/src/metabase/parameters/params.clj+++ b/src/metabase/parameters/params.clj@@ -27,6 +27,7 @@[metabase.util.log :as log][metabase.util.malli :as mu][metabase.util.malli.registry :as mr]+ [metabase.util.malli.schema :as ms][methodical.core :as methodical][toucan2.core :as t2]))@@ -471,3 +472,27 @@To get these IDs broken out by the Param ID that references them, use [[card->template-tag-param-id->field-ids]]."[card :- [:maybe :map]](some-> card :dataset_query not-empty lib-be/normalize-query lib/all-template-tag-field-ids not-empty))++(def ^:private ParamWithMapping+ [:map+ [:id ms/NonBlankString]+ [:name ms/NonBlankString]+ [:mappings [:maybe [:set ::parameters.schema/parameter-mapping-with-dashcard]]]])++;; Kept out of [[metabase.dashboards.models.dashboard]] so the query processor can use it without+;; depending on the Dashboard model.+(mu/defn dashboard->resolved-params :- [:map-of ms/NonBlankString ParamWithMapping]+ "Return map of Dashboard parameter key -> param with resolved `:mappings` (see the `:resolved-params` hydration+ in [[metabase.dashboards.models.dashboard]] for an example). Callers that only need the mappings (e.g. the QP) can+ pass slim dashcards instead of paying for the full hydration."+ [dashboard :- [:map+ [:parameters [:maybe [:sequential :map]]]+ [:dashcards [:maybe [:sequential [:map+ [:parameter_mappings [:maybe [:sequential :map]]]]]]]]]+ (let [param-key->mappings (apply+ merge-with set/union+ (for [dashcard (:dashcards dashboard)+ param (:parameter_mappings dashcard)]+ {(:parameter_id param) #{(assoc param :dashcard dashcard)}}))]+ (into {} (for [{param-key :id, :as param} (:parameters dashboard)]+ [(u/qualified-name param-key) (assoc param :mappings (get param-key->mappings param-key))]))))diff --git a/src/metabase/permissions/models/collection/graph.clj b/src/metabase/permissions/models/collection/graph.cljindex e70f0b5d937..1ac014f4dca 100644--- a/src/metabase/permissions/models/collection/graph.clj+++ b/src/metabase/permissions/models/collection/graph.clj@@ -6,6 +6,7 @@[clojure.set :as set][metabase.api.common :as api][metabase.audit-app.core :as audit]+ [metabase.collections.models.collection.root :as collection.root][metabase.permissions.db :as permissions.db][metabase.permissions.models.collection-permission-graph-revision :as c-perm-revision][metabase.permissions.models.permissions :as perms]@@ -51,11 +52,6 @@;;; -------------------------------------------------- Fetch Graph ---------------------------------------------------;;;-(defn- root-collection- "Requiring resolve to break a circular dependency"- []- (var-get (requiring-resolve 'metabase.collections.models.collection/root-collection)))-(defn- collection-permission-graph"Return the permission graph for the collections with id in `collection-ids` and the root collection."[group-perms]@@ -155,7 +151,7 @@collection-id :- [:or [:= :root] ms/PositiveInt]new-collection-perms :- CollectionPermissions](let [collection-id (if (= collection-id :root)- (assoc (root-collection) :namespace collection-namespace)+ (assoc collection.root/root-collection :namespace collection-namespace)collection-id)];; remove whatever entry is already there (if any) and add a new entry if applicable(perms/revoke-collection-permissions! group-id collection-id)diff --git a/src/metabase/permissions/models/permissions.clj b/src/metabase/permissions/models/permissions.cljindex 09e664d5100..b6bba7da1b6 100644--- a/src/metabase/permissions/models/permissions.clj+++ b/src/metabase/permissions/models/permissions.clj@@ -162,6 +162,7 @@[clojure.string :as str][metabase.api.common :as api][metabase.audit-app.core :as audit]+ [metabase.collections.models.collection.root :as collection.root][metabase.config.core :as config][metabase.models.interface :as mi][metabase.permissions.db :as permissions.db]@@ -327,7 +328,7 @@[_model m](if-let [collection-id (:collection_id m)](mi/can-write? (permissions.db/collection collection-id))- (mi/can-write? (var-get (requiring-resolve 'metabase.collections.models.collection/root-collection)))))+ (mi/can-write? collection.root/root-collection)));;; +----------------------------------------------------------------------------------------------------------------+;;; | ENTITY + LIFECYCLE |diff --git a/src/metabase/permissions/validation.clj b/src/metabase/permissions/validation.cljindex 33746301115..1684ab37ad1 100644--- a/src/metabase/permissions/validation.clj+++ b/src/metabase/permissions/validation.clj@@ -3,6 +3,7 @@[clojure.string :as str][metabase.api.common :as api][metabase.config.core :as config]+ [metabase.permissions.models.permissions :as perms][metabase.premium-features.core :as premium-features][metabase.util.i18n :refer [tru]]))@@ -13,10 +14,9 @@(check-has-application-permission perm-type true))([perm-type require-superuser?]- (if-let [f (and (premium-features/enable-advanced-permissions?)- config/ee-available?- (requiring-resolve 'metabase-enterprise.advanced-permissions.common/current-user-has-application-permissions?))]- (api/check-403 (f perm-type))+ (if (and (premium-features/enable-advanced-permissions?)+ config/ee-available?)+ (api/check-403 (perms/current-user-has-application-permissions? perm-type))(when require-superuser?(api/check-superuser)))))diff --git a/src/metabase/premium_features/token_check.clj b/src/metabase/premium_features/token_check.cljindex f4a20616f04..a0cfa5e11aa 100644--- a/src/metabase/premium_features/token_check.clj+++ b/src/metabase/premium_features/token_check.clj@@ -73,7 +73,7 @@(let [f (fn []{:post [(integer? %)]}(log/debug (u/colorize :yellow "GETTING ACTIVE USER COUNT!"))- (assert ((requiring-resolve 'metabase.app-db.core/db-is-set-up?)) "Metabase DB is not yet set up")+ (assert (app-db/db-is-set-up?) "Metabase DB is not yet set up");; force this to use a new Connection, it seems to be getting called in situations where the Connection;; is from a different thread and is invalid by the time we get to use it(let [result (binding [t2.conn/*current-connectable* nil]@@ -90,7 +90,7 @@(defn -active-users-count"Getter for the [[metabase.premium-features.settings/active-users-count]] Setting."[]- (if-not ((requiring-resolve 'metabase.app-db.core/db-is-set-up?))+ (if-not (app-db/db-is-set-up?)0(locking-active-user-count)))@@ -319,7 +319,7 @@{:pass-thru true})));; important to not count these errors against the circuit breaker. These are not the types of errors we need;; to circuit break. (#65294)- (when-not ((requiring-resolve 'metabase.app-db.core/db-is-set-up?))+ (when-not (app-db/db-is-set-up?)(throw (ex-info "Metabase DB is not yet set up"{:cause :token-check/app-db-not-ready})))(locking lockdiff --git a/src/metabase/pulse/events/report_timezone_updated.clj b/src/metabase/pulse/events/report_timezone_updated.cljindex c4f7e75a9e6..ba0b3db27c6 100644--- a/src/metabase/pulse/events/report_timezone_updated.clj+++ b/src/metabase/pulse/events/report_timezone_updated.clj@@ -1,7 +1,7 @@(ns metabase.pulse.events.report-timezone-updated(:require[metabase.events.core :as events]- [metabase.pulse.task.send-pulses :as send-pulses]+ [metabase.pulse.task.send-pulses-trigger :as task.send-pulses-trigger][methodical.core :as methodical]))(events/derive! ::event :metabase/event)@@ -10,4 +10,4 @@(methodical/defmethod events/publish-event! ::event"When the report-timezone Setting is updated, update the timezone of all SendPulse triggers."[_topic _event]- (send-pulses/update-send-pulse-triggers-timezone!))+ (task.send-pulses-trigger/update-send-pulse-triggers-timezone!))diff --git a/src/metabase/pulse/models/pulse_channel.clj b/src/metabase/pulse/models/pulse_channel.cljindex ccc48a1e32e..bc93bbefa27 100644--- a/src/metabase/pulse/models/pulse_channel.clj+++ b/src/metabase/pulse/models/pulse_channel.clj@@ -5,6 +5,7 @@[metabase.models.interface :as mi][metabase.notification.models :as notification.models][metabase.pulse.db :as pulse.db]+ [metabase.pulse.task.send-pulses-trigger :as task.send-pulses-trigger][metabase.util :as u][metabase.util.i18n :refer [tru]][methodical.core :as methodical]@@ -139,10 +140,6 @@{:email email})(get pcid->recipients (:id pc))))))))-(defn- update-send-pulse-trigger-if-needed!- [& args]- (apply (requiring-resolve 'metabase.pulse.task.send-pulses/update-send-pulse-trigger-if-needed!) args))-(def ^:dynamic *archive-parent-pulse-when-last-channel-is-deleted*"Should we automatically archive a Pulse when its last `PulseChannel` is deleted? Normally we do, but this is disabledin [[update-notification-channels!]] which creates/deletes/updates several channels sequentially."@@ -158,7 +155,7 @@(pulse.db/update-pulse! pulse-id {:archived true}))));; it's best if this is done in after-delete, but toucan2 doesn't support that yet See toucan2#70S;; remove this pulse from its existing trigger- (update-send-pulse-trigger-if-needed! pulse-id pulse-channel :remove-pc-ids #{(:id pulse-channel)}))+ (task.send-pulses-trigger/update-send-pulse-trigger-if-needed! pulse-id pulse-channel :remove-pc-ids #{(:id pulse-channel)}))(defn validate-email-domains"For channels that are being sent to raw email addresses: check that the domains in the emails are allowed by@@ -204,7 +201,7 @@[{:keys [pulse_id id] :as pulse-channel}](u/prog1 pulse-channel(when (:enabled pulse-channel)- (update-send-pulse-trigger-if-needed! pulse_id pulse-channel :add-pc-ids #{id}))))+ (task.send-pulses-trigger/update-send-pulse-trigger-if-needed! pulse_id pulse-channel :add-pc-ids #{id}))))(t2/define-before-update :model/PulseChannel[{:keys [pulse_id id] :as pulse-channel}]@@ -214,17 +211,17 @@;; better be done in after-update, but t2/changes isn't available in after-update yet See toucan2#129(when (some #(contains? #{:schedule_type :schedule_hour :schedule_day :schedule_frame} %) (keys changes));; need to remove this PC from the existing trigger- (update-send-pulse-trigger-if-needed! pulse_id (t2/original pulse-channel)- :remove-pc-ids #{(:id pulse-channel)})+ (task.send-pulses-trigger/update-send-pulse-trigger-if-needed! pulse_id (t2/original pulse-channel)+ :remove-pc-ids #{(:id pulse-channel)});; create a new PC with the updated schedule- (update-send-pulse-trigger-if-needed! pulse_id pulse-channel- :add-pc-ids #{id}))+ (task.send-pulses-trigger/update-send-pulse-trigger-if-needed! pulse_id pulse-channel+ :add-pc-ids #{id}))(when (contains? changes :enabled)(if (:enabled changes)- (update-send-pulse-trigger-if-needed! pulse_id pulse-channel- :add-pc-ids #{(:id pulse-channel)})- (update-send-pulse-trigger-if-needed! pulse_id (t2/original pulse-channel)- :remove-pc-ids #{(:id pulse-channel)}))))+ (task.send-pulses-trigger/update-send-pulse-trigger-if-needed! pulse_id pulse-channel+ :add-pc-ids #{(:id pulse-channel)})+ (task.send-pulses-trigger/update-send-pulse-trigger-if-needed! pulse_id (t2/original pulse-channel)+ :remove-pc-ids #{(:id pulse-channel)}))))(validate-email-domains (mi/changes-with-pk pulse-channel)));; ## Persistence Functionsdiff --git a/src/metabase/pulse/send.clj b/src/metabase/pulse/send.cljindex 50ee7155e3f..791a0dc58f9 100644--- a/src/metabase/pulse/send.clj+++ b/src/metabase/pulse/send.clj@@ -2,6 +2,7 @@"Code related to sending Pulses (Alerts or Dashboard Subscriptions)."(:require[metabase.models.interface :as mi]+ [metabase.notification.core :as notification][metabase.pulse.db :as pulse.db][metabase.pulse.models.pulse :as models.pulse][metabase.task-history.core :as task-history]@@ -98,8 +99,6 @@(update :schedule_frame maybe-name)))}]:handlers [(get-notification-handler pulse-channel)]}))-(def ^:private send-notification! (requiring-resolve 'metabase.notification.core/send-notification!))-(defn- send-pulse!*[{:keys [channels channel-ids] :as pulse} dashboard async?](let [;; `channel-ids` is the set of channels to send to now, so only send to those. Note the whole set of channels@@ -108,7 +107,7 @@channels)](doseq [pulse-channel channels](try- (send-notification! (notification-info pulse dashboard pulse-channel) :notification/sync? (not async?))+ (notification/send-notification! (notification-info pulse dashboard pulse-channel) :notification/sync? (not async?))(catch Exception e(log/errorf "[Pulse %d] Error sending to %s channel: %s" (:id pulse) (:channel_type pulse-channel) (ex-message e)))))nil))diff --git a/src/metabase/pulse/task/send_pulses.clj b/src/metabase/pulse/task/send_pulses.cljindex 2f7e017ec04..5fb609de8fb 100644--- a/src/metabase/pulse/task/send_pulses.clj+++ b/src/metabase/pulse/task/send_pulses.clj@@ -9,44 +9,22 @@[clojure.string :as str][clojurewerkz.quartzite.conversion :as qc][clojurewerkz.quartzite.jobs :as jobs]- [clojurewerkz.quartzite.schedule.cron :as cron][clojurewerkz.quartzite.triggers :as triggers]- [metabase.driver :as driver][metabase.pulse.db :as pulse.db][metabase.pulse.models.pulse :as models.pulse][metabase.pulse.send :as pulse.send]- [metabase.query-processor.timezone :as qp.timezone]+ [metabase.pulse.task.send-pulses-trigger :as task.send-pulses-trigger][metabase.task-history.core :as task-history][metabase.task.core :as task][metabase.tracing.core :as tracing]- [metabase.util.cron :as u.cron]- [metabase.util.log :as log]- [metabase.util.malli :as mu])+ [metabase.util.log :as log])(:import- (java.util TimeZone)- (org.quartz CronTrigger DisallowConcurrentExecution JobExecutionContext TriggerKey)))+ (org.quartz DisallowConcurrentExecution JobExecutionContext)))(set! *warn-on-reflection* true);;; ------------------------------------------------ Job: SendPulse -----------------------------------------------------(def ^:private send-pulse-job-key (jobs/key "metabase.task.send-pulses.send-pulse.job"))--(defn- send-pulse-trigger-key- ^TriggerKey [pulse-id schedule-map]- (triggers/key (format "metabase.task.send-pulse.trigger.%d.%s"- pulse-id (-> (select-keys schedule-map u.cron/schedule-keys)- u.cron/schedule-map->cron-string- (str/replace " " "_")))))--(defn- send-pulse-trigger-key->info- [trigger-key]- (let [[_ pulse-id schedule-str] (re-matches #"metabase\.task\.send-pulse\.trigger\.(\d+)\.(.*)" trigger-key)]- {:pulse-id (parse-long pulse-id)- :schedule-map (-> schedule-str- (str/replace "_" " ")- u.cron/cron-string->schedule-map)}))-(defn- send-pulse (ex-message e))))))-(defn- send-trigger-timezone- []- (or (driver/report-timezone)- (qp.timezone/system-timezone-id)- "UTC"))--(mu/defn- send-pulse-trigger- "Build a Quartz trigger to send a pulse to a list of channel-ids."- ^CronTrigger- ([pulse-id :- pos-int?- schedule-map :- u.cron/ScheduleMap- pc-ids :- [:set pos-int?]- timezone :- :string]- (send-pulse-trigger pulse-id schedule-map pc-ids timezone 6))- ([pulse-id :- pos-int?- schedule-map :- u.cron/ScheduleMap- pc-ids :- [:set pos-int?]- timezone :- :string- priority :- pos-int?]- (triggers/build- (triggers/with-identity (send-pulse-trigger-key pulse-id schedule-map))- (triggers/for-job send-pulse-job-key)- (triggers/using-job-data {"pulse-id" pulse-id- "channel-ids" pc-ids})- (triggers/with-schedule- (cron/schedule- (cron/cron-schedule (u.cron/schedule-map->cron-string (select-keys schedule-map u.cron/schedule-keys)))- (cron/in-time-zone (TimeZone/getTimeZone ^String timezone))- ;; If the trigger is misfired, fire it immediately and proceed with the next scheduled time.- ;; TODO: upon testing, look like re-firing on startup is not working as expected- ;;- ;; See https://www.nurkiewicz.com/2012/04/quartz-scheduler-misfire-instructions.html for more info- (cron/with-misfire-handling-instruction-fire-and-proceed)))- (triggers/with-priority priority))))-; Clearing pulse channels is not done synchronously in order to support undoing feature.(defn- clear-pulse-channels-no-recipients!"Delete PulseChannels that have no recipients and no channel set for a pulse, returns the channel ids that were deleted."@@ -134,20 +77,6 @@(send-pulse! pulse-id to-send-enabled-channel-ids)(log/infof "Skip sending pulse %d because all channels have no recipients" pulse-id))))-(defn update-send-pulse-triggers-timezone!- "Update the timezone of all SendPulse triggers if the report timezone changes."- []- (let [triggers (-> send-pulse-job-key task/job-info :triggers)- new-timezone (send-trigger-timezone)]- (doseq [trigger triggers- :when (not= new-timezone (:timezone trigger))] ; skip if timezone is the same- (let [trigger-key (:key trigger)- channel-ids (get-in trigger [:data "channel-ids"])- {:keys [pulse-id- schedule-map]} (send-pulse-trigger-key->info trigger-key)]- (log/infof "Updating timezone of trigger %s to %s. Was: %s" trigger-key new-timezone (:timezone trigger))- (task/reschedule-trigger! (send-pulse-trigger pulse-id schedule-map channel-ids new-timezone (:priority trigger)))))))-(task/defjob ^{:doc "Triggers that send a pulse to a list of channels at a specific time"}SendPulse[context]@@ -171,73 +100,6 @@pulse-id trigger-key scheduled-fire-time fire-time recovering? refire-count scheduler-id)(send-pulse!* pulse-id channel-ids))))-(declare update-send-pulse-trigger-if-needed!)--(defn- active-dashsub-pcs- []- (pulse.db/active-dashboard-subscription-channels))--(defn init-dashboard-subscription-triggers!- "Update send pulse triggers for all active pulses.- Called once when Metabase starts up to create triggers for all existing PulseChannels"- []- (assert (task/scheduler) "Scheduler must be started before initializing SendPulse triggers")- (let [trigger-slot->pc-ids (as-> (active-dashsub-pcs) results- (group-by #(select-keys % [:pulse_id :schedule_type :schedule_day :schedule_hour :schedule_frame]) results)- (update-vals results #(map :id %)))]- (doseq [[{:keys [pulse_id] :as schedule-map} pc-ids] trigger-slot->pc-ids]- (update-send-pulse-trigger-if-needed! pulse_id schedule-map :add-pc-ids (set pc-ids)))))--;;; --------------------------------------------- Helpers ---------------------------------------------;; called by PulseChannel hooks-(defn update-send-pulse-trigger-if-needed!- "Update send pulse trigger of a pulse for a specific schedule map with new pulse channel ids.-- Send Pulse triggers are grouped by pulse id and schedule time, meaning PulseChannels of a Pulse that scheduled- to run at the same time of will be send together.- This function will updates the corresponding trigger if PulseChannels changes.-- * To add 2 pulse channels to a trigger- (update-send-pulse-trigger-if-needed! pulse-id schedule-map :add-pc-ids #{1 2}))-- * To remove 2 pulse channels from a trigger- (update-send-pulse-trigger-if-needed! pulse-id schedule-map :remove-pc-ids #{1 2}))"- [pulse-id schedule-map & {:keys [add-pc-ids remove-pc-ids]}]- (let [schedule-map (update-vals (select-keys schedule-map u.cron/schedule-keys)- #(if (keyword? %)- (name %)- %))- trigger-key (send-pulse-trigger-key pulse-id schedule-map)- ;; there should be at most one existing trigger- existing-trigger (->> (-> send-pulse-job-key task/job-info :triggers)- (filter #(= (:key %) (.getName ^TriggerKey trigger-key)))- first)- existing-pc-ids (some-> existing-trigger :data (get "channel-ids") set)- new-pc-ids (if (some? existing-pc-ids)- (cond-> existing-pc-ids- (some? add-pc-ids) (set/union existing-pc-ids (set add-pc-ids))- (some? remove-pc-ids) (set/difference (set remove-pc-ids)))- (set add-pc-ids))]- (cond- ;; no op when new-pc-ids doesn't change- (= new-pc-ids existing-pc-ids) nil-- ;; delete if no new pc-ids and there is an existing trigger- (and (empty? new-pc-ids)- (some? existing-pc-ids))- (do- (log/infof "Deleting trigger %s for pulse %d" trigger-key pulse-id)- (task/delete-trigger! trigger-key))-- ;; delete then create if pc ids changes- (and (seq new-pc-ids)- (not= new-pc-ids existing-pc-ids))- (do- (log/infof "Updating Send Pulse trigger %s for pulse %d with new pc-ids: %s, was: %s " trigger-key pulse-id new-pc-ids existing-pc-ids)- (task/delete-trigger! trigger-key)- (task/add-trigger! (send-pulse-trigger pulse-id schedule-map new-pc-ids (send-trigger-timezone)))))))-(task/defjob^{:doc"Find all notification subscriptions with cron schedules and create a trigger for each.@@ -252,13 +114,13 @@InitSendPulseTriggers[_context](log/info "Initializing SendPulse triggers for dashboard subscriptions")- (init-dashboard-subscription-triggers!))+ (task.send-pulses-trigger/init-dashboard-subscription-triggers!));;; -------------------------------------------------- Task init ------------------------------------------------(defmethod task/init! ::SendPulses [_](let [send-pulse-job (jobs/build- (jobs/with-identity send-pulse-job-key)+ (jobs/with-identity task.send-pulses-trigger/send-pulse-job-key)(jobs/with-description "Send Pulse")(jobs/of-type SendPulse)(jobs/store-durably))diff --git a/src/metabase/pulse/task/send_pulses_trigger.clj b/src/metabase/pulse/task/send_pulses_trigger.cljnew file mode 100644index 00000000000..f46959a056e--- /dev/null+++ b/src/metabase/pulse/task/send_pulses_trigger.clj@@ -0,0 +1,154 @@+(ns metabase.pulse.task.send-pulses-trigger+ "Quartz trigger lifecycle for the SendPulse job.++ Separate from [[metabase.pulse.task.send-pulses]] so that PulseChannel model hooks can (un)schedule triggers: the job+ body over there loads the sending machinery, which reads the models."+ (:require+ [clojure.set :as set]+ [clojure.string :as str]+ [clojurewerkz.quartzite.jobs :as jobs]+ [clojurewerkz.quartzite.schedule.cron :as cron]+ [clojurewerkz.quartzite.triggers :as triggers]+ [metabase.driver :as driver]+ [metabase.pulse.db :as pulse.db]+ [metabase.query-processor.timezone :as qp.timezone]+ [metabase.task.core :as task]+ [metabase.util.cron :as u.cron]+ [metabase.util.log :as log]+ [metabase.util.malli :as mu])+ (:import+ (java.util TimeZone)+ (org.quartz CronTrigger TriggerKey)))++(set! *warn-on-reflection* true)++(def send-pulse-job-key+ "Key of the Quartz job that sends a pulse when one of its triggers fires."+ (jobs/key "metabase.task.send-pulses.send-pulse.job"))++(defn- send-pulse-trigger-key+ ^TriggerKey [pulse-id schedule-map]+ (triggers/key (format "metabase.task.send-pulse.trigger.%d.%s"+ pulse-id (-> (select-keys schedule-map u.cron/schedule-keys)+ u.cron/schedule-map->cron-string+ (str/replace " " "_")))))++(defn- send-pulse-trigger-key->info+ [trigger-key]+ (let [[_ pulse-id schedule-str] (re-matches #"metabase\.task\.send-pulse\.trigger\.(\d+)\.(.*)" trigger-key)]+ {:pulse-id (parse-long pulse-id)+ :schedule-map (-> schedule-str+ (str/replace "_" " ")+ u.cron/cron-string->schedule-map)}))++(defn- send-trigger-timezone+ []+ (or (driver/report-timezone)+ (qp.timezone/system-timezone-id)+ "UTC"))++(mu/defn- send-pulse-trigger+ "Build a Quartz trigger to send a pulse to a list of channel-ids."+ ^CronTrigger+ ([pulse-id :- pos-int?+ schedule-map :- u.cron/ScheduleMap+ pc-ids :- [:set pos-int?]+ timezone :- :string]+ (send-pulse-trigger pulse-id schedule-map pc-ids timezone 6))+ ([pulse-id :- pos-int?+ schedule-map :- u.cron/ScheduleMap+ pc-ids :- [:set pos-int?]+ timezone :- :string+ priority :- pos-int?]+ (triggers/build+ (triggers/with-identity (send-pulse-trigger-key pulse-id schedule-map))+ (triggers/for-job send-pulse-job-key)+ (triggers/using-job-data {"pulse-id" pulse-id+ "channel-ids" pc-ids})+ (triggers/with-schedule+ (cron/schedule+ (cron/cron-schedule (u.cron/schedule-map->cron-string (select-keys schedule-map u.cron/schedule-keys)))+ (cron/in-time-zone (TimeZone/getTimeZone ^String timezone))+ ;; If the trigger is misfired, fire it immediately and proceed with the next scheduled time.+ ;; TODO: upon testing, look like re-firing on startup is not working as expected+ ;;+ ;; See https://www.nurkiewicz.com/2012/04/quartz-scheduler-misfire-instructions.html for more info+ (cron/with-misfire-handling-instruction-fire-and-proceed)))+ (triggers/with-priority priority))))++(defn update-send-pulse-triggers-timezone!+ "Update the timezone of all SendPulse triggers if the report timezone changes."+ []+ (let [triggers (-> send-pulse-job-key task/job-info :triggers)+ new-timezone (send-trigger-timezone)]+ (doseq [trigger triggers+ :when (not= new-timezone (:timezone trigger))] ; skip if timezone is the same+ (let [trigger-key (:key trigger)+ channel-ids (get-in trigger [:data "channel-ids"])+ {:keys [pulse-id+ schedule-map]} (send-pulse-trigger-key->info trigger-key)]+ (log/infof "Updating timezone of trigger %s to %s. Was: %s" trigger-key new-timezone (:timezone trigger))+ (task/reschedule-trigger! (send-pulse-trigger pulse-id schedule-map channel-ids new-timezone (:priority trigger)))))))++;; called by PulseChannel hooks+(defn update-send-pulse-trigger-if-needed!+ "Update send pulse trigger of a pulse for a specific schedule map with new pulse channel ids.++ Send Pulse triggers are grouped by pulse id and schedule time, meaning PulseChannels of a Pulse that scheduled+ to run at the same time of will be send together.+ This function will updates the corresponding trigger if PulseChannels changes.++ * To add 2 pulse channels to a trigger+ (update-send-pulse-trigger-if-needed! pulse-id schedule-map :add-pc-ids #{1 2}))++ * To remove 2 pulse channels from a trigger+ (update-send-pulse-trigger-if-needed! pulse-id schedule-map :remove-pc-ids #{1 2}))"+ [pulse-id schedule-map & {:keys [add-pc-ids remove-pc-ids]}]+ (let [schedule-map (update-vals (select-keys schedule-map u.cron/schedule-keys)+ #(if (keyword? %)+ (name %)+ %))+ trigger-key (send-pulse-trigger-key pulse-id schedule-map)+ ;; there should be at most one existing trigger+ existing-trigger (->> (-> send-pulse-job-key task/job-info :triggers)+ (filter #(= (:key %) (.getName ^TriggerKey trigger-key)))+ first)+ existing-pc-ids (some-> existing-trigger :data (get "channel-ids") set)+ new-pc-ids (if (some? existing-pc-ids)+ (cond-> existing-pc-ids+ (some? add-pc-ids) (set/union existing-pc-ids (set add-pc-ids))+ (some? remove-pc-ids) (set/difference (set remove-pc-ids)))+ (set add-pc-ids))]+ (cond+ ;; no op when new-pc-ids doesn't change+ (= new-pc-ids existing-pc-ids) nil++ ;; delete if no new pc-ids and there is an existing trigger+ (and (empty? new-pc-ids)+ (some? existing-pc-ids))+ (do+ (log/infof "Deleting trigger %s for pulse %d" trigger-key pulse-id)+ (task/delete-trigger! trigger-key))++ ;; delete then create if pc ids changes+ (and (seq new-pc-ids)+ (not= new-pc-ids existing-pc-ids))+ (do+ (log/infof "Updating Send Pulse trigger %s for pulse %d with new pc-ids: %s, was: %s " trigger-key pulse-id new-pc-ids existing-pc-ids)+ (task/delete-trigger! trigger-key)+ (task/add-trigger! (send-pulse-trigger pulse-id schedule-map new-pc-ids (send-trigger-timezone)))))))++(defn- active-dashsub-pcs+ []+ (pulse.db/active-dashboard-subscription-channels))++(defn init-dashboard-subscription-triggers!+ "Update send pulse triggers for all active pulses.+ Called once when Metabase starts up to create triggers for all existing PulseChannels"+ []+ (assert (task/scheduler) "Scheduler must be started before initializing SendPulse triggers")+ (let [trigger-slot->pc-ids (as-> (active-dashsub-pcs) results+ (group-by #(select-keys % [:pulse_id :schedule_type :schedule_day :schedule_hour :schedule_frame]) results)+ (update-vals results #(map :id %)))]+ (doseq [[{:keys [pulse_id] :as schedule-map} pc-ids] trigger-slot->pc-ids]+ (update-send-pulse-trigger-if-needed! pulse_id schedule-map :add-pc-ids (set pc-ids)))))diff --git a/src/metabase/queries/card.clj b/src/metabase/queries/card.cljindex a09567112e7..547e40a517c 100644--- a/src/metabase/queries/card.clj+++ b/src/metabase/queries/card.clj@@ -8,8 +8,8 @@[metabase.parameters.field :as parameters.field][metabase.parameters.params :as params][metabase.parameters.schema :as parameters.schema]- [metabase.queries.models.card :as card][metabase.queries.schema :as queries.schema]+ [metabase.query-processor.card :as qp.card][metabase.util :as u][metabase.util.i18n :refer [tru]][metabase.util.malli :as mu]@@ -21,7 +21,7 @@(u/prog1 (m/find-first #(= (:id %) param-key)(or (seq (:parameters card));; some older cards or cards in e2e just use the template tags on native queries- (card/template-tag-parameters card)))+ (qp.card/template-tag-parameters card)))(when-not <>(throw (ex-info (tru "Card does not have a parameter with the ID {0}" (pr-str param-key)){:status-code 400})))))@@ -46,7 +46,7 @@slug->param (into {}(map (juxt (comp keyword :slug) identity))(or (seq (:parameters card))- (card/template-tag-parameters card)))]+ (qp.card/template-tag-parameters card)))](vec (for [[slug value] slug->value;; an empty collection means "no value", as it does in `validate-and-merge-params`, and a parameter;; with no value constrains nothing whether or not it still existsdiff --git a/src/metabase/queries/core.clj b/src/metabase/queries/core.cljindex 3fcadc1ad7e..42d3b940676 100644--- a/src/metabase/queries/core.clj+++ b/src/metabase/queries/core.clj@@ -10,6 +10,7 @@[metabase.queries.models.query][metabase.queries.models.stored-result][metabase.queries.models.stored-result-use]+ [metabase.query-processor.card][potemkin :as p]))(comment metabase.queries.cached-result/keep-me@@ -21,7 +22,8 @@metabase.queries.models.parameter-card/keep-memetabase.queries.models.query/keep-memetabase.queries.models.stored-result/keep-me- metabase.queries.models.stored-result-use/keep-me)+ metabase.queries.models.stored-result-use/keep-me+ metabase.query-processor.card/keep-me)(p/import-vars[metabase.queries.card@@ -47,7 +49,6 @@maybe-unverify!model-supports-implicit-actions?model?- parameter-template-tag?sole-dashboard-idstarting-card-schema-versionupdate-card!@@ -73,7 +74,9 @@cached-dataset][metabase.queries.models.stored-result-useassert-can-view-card-snapshots!- carry-pairings-for-document!])+ carry-pairings-for-document!]+ [metabase.query-processor.card+ parameter-template-tag?]);; the re-exported var carries the docstring; kondo can't see through import-def#_{:clj-kondo/ignore [:missing-docstring]}@@ -81,7 +84,7 @@;; the re-exported var carries the docstring; kondo can't see through import-def#_{:clj-kondo/ignore [:missing-docstring]}-(p/import-def metabase.queries.models.card/template-tag-parameters card-template-tag-parameters)+(p/import-def metabase.query-processor.card/template-tag-parameters card-template-tag-parameters);; the re-exported var carries the docstring; kondo can't see through import-def#_{:clj-kondo/ignore [:missing-docstring]}diff --git a/src/metabase/queries/models/card.clj b/src/metabase/queries/models/card.cljindex 731e13dd91e..954ceb77746 100644--- a/src/metabase/queries/models/card.clj+++ b/src/metabase/queries/models/card.clj@@ -23,13 +23,11 @@[metabase.lib.schema :as lib.schema][metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.metadata :as lib.schema.metadata]- [metabase.lib.schema.template-tag :as lib.schema.template-tag][metabase.metrics.core :as metrics][metabase.models.interface :as mi][metabase.models.serialization :as serdes][metabase.parameters.core :as parameters][metabase.parameters.params :as params]- [metabase.parameters.schema :as parameters.schema][metabase.permissions.core :as perms][metabase.premium-features.core :refer [defenterprise]][metabase.public-sharing.core :as public-sharing]@@ -42,6 +40,7 @@[metabase.search.core :as search][metabase.settings.core :as setting][metabase.staleness.core :as staleness]+ [metabase.sync.field-values :as sync.field-values][metabase.util :as u][metabase.util.embed :refer [maybe-populate-initially-published-at]][metabase.util.honey-sql-2 :as h2x]@@ -335,48 +334,6 @@(when database-id{:database_id database-id})))))))))-;;; TODO -- move this to [[metabase.query-processor.card]] or Lib so the logic can be shared between the backend and-;;; frontend (?)-;;;-;;; NOTE: this should mirror `getTemplateTagParameters` in frontend/src/metabase-lib/parameters/utils/template-tags.ts-;;; If this function moves you should update the comment that links to this one (#40013)-;;;-(mu/defn parameter-template-tag? :- :boolean- "Whether a parameter is created for this template tag, as opposed to tags that splice content into the query itself,- like snippets, card references, and tables."- [{tag-type :type, widget-type :widget-type} :- [:maybe ::lib.schema.template-tag/template-tag]]- (boolean- (and tag-type- (or (contains? lib.schema.template-tag/raw-value-template-tag-types tag-type)- (= tag-type :temporal-unit)- (and (= tag-type :dimension) widget-type (not= widget-type :none))))))--;;; TODO -- does this belong HERE or in the `parameters` module?-(mu/defn template-tag-parameters :- ::parameters.schema/parameters- "Transforms native query's `template-tags` into `parameters`.- An older style was to not include `:template-tags` onto cards as parameters. I think this is a mistake and they- should always be there. Apparently lots of e2e tests are sloppy about this so this is included as a convenience."- [card :- [:maybe ::queries.schema/card]]- (for [{tag-type :type, widget-type :widget-type, :as tag} (some-> card :dataset_query not-empty lib/all-template-tags)- :when (parameter-template-tag? tag)]- {:id (:id tag)- :type (or widget-type (case tag-type- :temporal-unit :temporal-unit- :date :date/single- :text :string/=- :number :number/=- :boolean :boolean/=- ;; fallback; should be unreachable since :when filters- ;; to raw-value-template-tag-types- :string/=))- :target (if (contains? #{:dimension :temporal-unit} tag-type)- [:dimension [:template-tag (:name tag)]]- [:variable [:template-tag (:name tag)]])- :name (:display-name tag)- :slug (:name tag)- :default (:default tag)- :required (boolean (:required tag))}))-(defn- check-field-filter-fields-are-from-correct-database"Check that all native query Field filter parameters reference Fields belonging to the Database the query pointsagainst. This is done when saving a Card. The goal here is to prevent people from saving Cards with invalid queries@@ -614,7 +571,7 @@"Is Now:" new-param-field-ids"Newly Added:" newly-added-param-field-ids);; Now update the FieldValues for the Fields referenced by this Card.- ((requiring-resolve 'metabase.sync.field-values/update-field-values-for-on-demand-dbs!) newly-added-param-field-ids)))))+ (sync.field-values/update-field-values-for-on-demand-dbs! newly-added-param-field-ids)))));; updating a model dataset query to not support implicit actions will disable implicit actions if they exist(when (and (:dataset_query changes)(= (:type old-card-info) :model)@@ -821,7 +778,7 @@(u/prog1 card(when-let [field-ids (seq (params/card->template-tag-field-ids card))](log/info "Card references Fields in params:" field-ids)- ((requiring-resolve 'metabase.sync.field-values/update-field-values-for-on-demand-dbs!) field-ids))+ (sync.field-values/update-field-values-for-on-demand-dbs! field-ids))(parameter-card/upsert-or-delete-from-parameters! "card" (:id card) (:parameters card))))(defn- apply-dashboard-question-updates [card changes]diff --git a/src/metabase/query_permissions/impl.clj b/src/metabase/query_permissions/impl.cljindex 46facd4396f..24496b63f35 100644--- a/src/metabase/query_permissions/impl.clj+++ b/src/metabase/query_permissions/impl.clj@@ -20,8 +20,10 @@[metabase.query-permissions.db :as query-permissions.db][metabase.query-processor.error-type :as qp.error-type][metabase.query-processor.interface :as qp.i]+ [metabase.query-processor.preprocess :as qp.preprocess];; legacy usage -- don't do things like this going forward^{:clj-kondo/ignore [:deprecated-namespace :discouraged-namespace]} [metabase.query-processor.store :as qp.store]+ [metabase.request.core :as request][metabase.util :as u][metabase.util.i18n :refer [tru]][metabase.util.log :as log]@@ -185,11 +187,8 @@;; ignore the current user for the purposes of calculating the permissions required to run the query. Don't want the;; preprocessing to fail because current user doesn't have permissions to run it when we're not trying to run it at;; all- (let [do-as-admin (requiring-resolve 'metabase.request.core/do-as-admin)- preprocess (requiring-resolve 'metabase.query-processor.preprocess/preprocess)]- (do-as-admin- (^:once fn* []- (preprocess query)))))+ (request/as-admin+ (qp.preprocess/preprocess query)))(defn- preprocess-without-per-user-lens"[[preprocess-query]], minus the preprocess middlewares that resolve the current user's data-accessdiff --git a/src/metabase/query_processor/card.clj b/src/metabase/query_processor/card.cljindex ea66b77f9d0..9979e89d700 100644--- a/src/metabase/query_processor/card.clj+++ b/src/metabase/query_processor/card.clj@@ -15,7 +15,7 @@[metabase.lib.schema.template-tag :as lib.schema.template-tag][metabase.parameters.schema :as parameters.schema][metabase.premium-features.core :refer [defenterprise]]- [metabase.queries.core :as queries]+ [metabase.queries.models.query :as queries.query][metabase.queries.schema :as queries.schema][metabase.query-processor :as qp][metabase.query-processor.error-type :as qp.error-type]@@ -48,7 +48,7 @@(defn- enrich-strategy [strategy query](case (:type strategy)- :ttl (let [et (queries/average-execution-time-ms (qp.util/query-hash query))]+ :ttl (let [et (queries.query/average-execution-time-ms (qp.util/query-hash query))](assoc strategy :avg-execution-ms (or et 0)))strategy))@@ -140,6 +140,49 @@here."false)+;;; These live here rather than in [[metabase.queries.models.card]] so the QP can use them without+;;; depending on the Card model.+(mu/defn parameter-template-tag? :- :boolean+ "Whether a parameter is created for this template tag, as opposed to tags that splice content into the query itself,+ like snippets, card references, and tables."+ [{tag-type :type, widget-type :widget-type} :- [:maybe ::lib.schema.template-tag/template-tag]]+ (boolean+ (and tag-type+ (or (contains? lib.schema.template-tag/raw-value-template-tag-types tag-type)+ (= tag-type :temporal-unit)+ (and (= tag-type :dimension) widget-type (not= widget-type :none))))))++;;; TODO -- move this to Lib so the logic can be shared between the backend and frontend (?)+;;;+;;; NOTE: this should mirror `getTemplateTagParameters` in frontend/src/metabase-lib/parameters/utils/template-tags.ts+;;; If this function moves you should update the comment that links to this one (#40013)+;;;+;;; TODO -- does this belong HERE or in the `parameters` module?+(mu/defn template-tag-parameters :- ::parameters.schema/parameters+ "Transforms native query's `template-tags` into `parameters`.+ An older style was to not include `:template-tags` onto cards as parameters. I think this is a mistake and they+ should always be there. Apparently lots of e2e tests are sloppy about this so this is included as a convenience."+ [card :- [:maybe ::queries.schema/card]]+ (for [{tag-type :type, widget-type :widget-type, :as tag} (some-> card :dataset_query not-empty lib/all-template-tags)+ :when (parameter-template-tag? tag)]+ {:id (:id tag)+ :type (or widget-type (case tag-type+ :temporal-unit :temporal-unit+ :date :date/single+ :text :string/=+ :number :number/=+ :boolean :boolean/=+ ;; fallback; should be unreachable since :when filters+ ;; to raw-value-template-tag-types+ :string/=))+ :target (if (contains? #{:dimension :temporal-unit} tag-type)+ [:dimension [:template-tag (:name tag)]]+ [:variable [:template-tag (:name tag)]])+ :name (:display-name tag)+ :slug (:name tag)+ :default (:default tag)+ :required (boolean (:required tag))}))+(mu/defn- card-template-tag-parameters"Template tag parameters that have been specified for the Card's `query` (`:dataset_query`), if any, returned as a mapin the format@@ -265,8 +308,8 @@is not present in the parameters.This function ensures that all template-tags are converted to parameters and added to card.parameters."[{:keys [parameters] :as card} :- ::queries.schema/card]- (let [template-tag-parameters (queries/card-template-tag-parameters card)- id->template-tags-parameter (m/index-by :id template-tag-parameters)+ (let [tag-parameters (template-tag-parameters card)+ id->template-tags-parameter (m/index-by :id tag-parameters)parameter-ids (into #{} (map :id) parameters);; Preserve the order of card.parameters, merging in template-tag infomerged-parameters (mapv (fn [param]@@ -278,7 +321,7 @@;; Append any template-tag parameters not already present in card.parameters(into merged-parameters(remove #(contains? parameter-ids (:id %)))- template-tag-parameters)))+ tag-parameters)))(mu/defn- enrich-parameters-from-card :- ::parameters.schema/parameters"Allow the FE to omit type and target for parameters by adding them from the card."diff --git a/src/metabase/query_processor/dashboard.clj b/src/metabase/query_processor/dashboard.cljindex 2542d1895cc..e73d87b8721 100644--- a/src/metabase/query_processor/dashboard.clj+++ b/src/metabase/query_processor/dashboard.clj@@ -5,12 +5,12 @@[clojure.string :as str][medley.core :as m][metabase.api.common :as api]- [metabase.dashboards.models.dashboard :as dashboard][metabase.dashboards.schema :as dashboards.schema][metabase.events.core :as events][metabase.lib-metric.core :as lib-metric][metabase.lib.core :as lib][metabase.lib.schema.id :as lib.schema.id]+ [metabase.parameters.params :as params][metabase.permissions.core :as perms][metabase.permissions.metric :as permissions.metric][metabase.query-processor.card :as qp.card]@@ -142,7 +142,7 @@(defn- dashboard-param-defaults"Construct parameter entries for any parameters with default values in `dashboard-param-id->param` as returned- by [[dashboard/dashboard->resolved-params]]."+ by [[params/dashboard->resolved-params]]."[dashboard-param-id->param card-id](into{}@@ -173,7 +173,7 @@dashcard-id (:id dashcard)_ (log/tracef "Resolving Dashboard %d Card %d query request parameters" dashboard-id card-id)request-params (some-> request-params not-empty (->> (lib/normalize ::dashboards.schema/parameters)))- resolved-params (dashboard/dashboard->resolved-params (assoc dashboard :dashcards [dashcard]))+ resolved-params (params/dashboard->resolved-params (assoc dashboard :dashcards [dashcard]))dashboard-param-id->param (into {};; remove the `:default` values from Dashboard params. We don't ACTUALLY want to;; use these values ourselves -- the expectation is that the frontend will passdiff --git a/src/metabase/query_processor/middleware/annotate.clj b/src/metabase/query_processor/middleware/annotate.cljindex 70859829ee4..c5625ce2ecf 100644--- a/src/metabase/query_processor/middleware/annotate.clj+++ b/src/metabase/query_processor/middleware/annotate.clj@@ -12,7 +12,7 @@[metabase.lib.schema :as lib.schema][metabase.lib.schema.metadata :as lib.schema.metadata][metabase.query-processor.debug :as qp.debug]- [metabase.query-processor.middleware.annotate.legacy-helper-fns]+ [metabase.query-processor.middleware.annotate.legacy-helper-fns :as annotate.legacy-helper-fns][metabase.query-processor.reducible :as qp.reducible][metabase.query-processor.schema :as qp.schema][metabase.util.malli :as mu]@@ -20,8 +20,6 @@[metabase.util.performance :refer [every? mapv empty? get-in]][potemkin :as p]))-(comment metabase.query-processor.middleware.annotate.legacy-helper-fns/keep-me)-(mr/def ::col[:map[:source {:optional true} ::lib.schema.metadata/column.legacy-source]@@ -159,6 +157,13 @@;;; convert drivers to MBQL 5.#_{:clj-kondo/ignore [:deprecated-var]}(p/import-vars- [metabase.query-processor.middleware.annotate.legacy-helper-fns- aggregation-name- merged-column-info])+ [annotate.legacy-helper-fns+ aggregation-name])++(mu/defn merged-column-info :- ::cols+ "Returns deduplicated and merged column metadata (`:cols`) for query results by combining (a) the initial results+ metadata returned by the driver's impl of `execute-reducible-query` and (b) column metadata inferred by logic in+ this namespace."+ {:deprecated "0.64.0"}+ [legacy-query {initial-cols :cols, :as _initial-metadata} :- [:maybe :map]]+ (expected-cols (annotate.legacy-helper-fns/legacy-query->mbql5-query legacy-query) initial-cols))diff --git a/src/metabase/query_processor/middleware/annotate/legacy_helper_fns.clj b/src/metabase/query_processor/middleware/annotate/legacy_helper_fns.cljindex 0b0846c7586..ef82686f02d 100644--- a/src/metabase/query_processor/middleware/annotate/legacy_helper_fns.clj+++ b/src/metabase/query_processor/middleware/annotate/legacy_helper_fns.clj@@ -41,6 +41,17 @@{:inner-query inner-query, :type qp.error-type/qp}e))))))+(mu/defn legacy-query->mbql5-query :- ::lib.schema/query+ "Convert a legacy outer `legacy-query` to an MBQL 5 query. Requires bound QP store."+ [legacy-query]+ (lib/query+ (qp.store/metadata-provider)+ ;; if this query has a `:native` query added to it already then remove that so we don't get schema validation+ ;; errors+ (cond-> legacy-query+ ((every-pred :native :query) legacy-query)+ (dissoc :native))))+(mu/defn aggregation-name :- ::lib.schema.common/non-blank-string"Return an appropriate aggregation name/alias *used inside a query* for an `:aggregation` subclause (an aggregationor expression). Takes an options map as schema won't support passing keypairs directly as a varargs.@@ -59,19 +70,3 @@#_{:clj-kondo/ignore [:deprecated-var]}(legacy-inner-query->mbql5-query legacy-inner-query)(lib/->mbql5 legacy-ag-clause)))))--(mu/defn merged-column-info :- :metabase.query-processor.middleware.annotate/cols- "Returns deduplicated and merged column metadata (`:cols`) for query results by combining (a) the initial results- metadata returned by the driver's impl of `execute-reducible-query` and (b) column metadata inferred by logic in- this namespace."- {:deprecated "0.64.0"}- [legacy-query {initial-cols :cols, :as _initial-metadata} :- [:maybe :map]]- (let [expected-cols (requiring-resolve 'metabase.query-processor.middleware.annotate/expected-cols)- mbql5-query (lib/query- (qp.store/metadata-provider)- ;; if this query has a `:native` query added to it already then remove that so we don't get- ;; schema validation errors- (cond-> legacy-query- ((every-pred :native :query) legacy-query)- (dissoc :native)))]- (expected-cols mbql5-query initial-cols)))diff --git a/src/metabase/query_processor/middleware/catch_exceptions.clj b/src/metabase/query_processor/middleware/catch_exceptions.cljindex 5a35488c5b0..86b212a3d50 100644--- a/src/metabase/query_processor/middleware/catch_exceptions.clj+++ b/src/metabase/query_processor/middleware/catch_exceptions.clj@@ -6,9 +6,11 @@[metabase.analytics-interface.core :as analytics][metabase.driver :as driver][metabase.lib.schema.common :as lib.schema.common]+ [metabase.query-processor.compile :as qp.compile][metabase.query-processor.error-type :as qp.error-type][metabase.query-processor.middleware.permissions :as qp.perms][metabase.query-processor.pipeline :as qp.pipeline]+ [metabase.query-processor.preprocess :as qp.preprocess][metabase.query-processor.schema :as qp.schema][metabase.util :as u][metabase.util.i18n :refer [trs]]@@ -132,9 +134,9 @@(qp query rff)(let [extra-info (delay{:native (u/ignore-exceptions- ((requiring-resolve 'metabase.query-processor.compile/compile) query))+ (qp.compile/compile query)):preprocessed (u/ignore-exceptions- ((requiring-resolve 'metabase.query-processor.preprocess/preprocess) query))})]+ (qp.preprocess/preprocess query))})](try(qp query rff)(catch Throwable ediff --git a/src/metabase/query_processor/middleware/permissions.clj b/src/metabase/query_processor/middleware/permissions.cljindex 8359d7c543e..89b6412f0a2 100644--- a/src/metabase/query_processor/middleware/permissions.clj+++ b/src/metabase/query_processor/middleware/permissions.clj@@ -9,12 +9,10 @@[metabase.permissions.core :as perms][metabase.premium-features.core :refer [defenterprise]][metabase.query-permissions.core :as query-perms]- [metabase.query-processor.pipeline :as qp.pipeline][metabase.query-processor.schema :as qp.schema];; the legacy QP pipeline still conveys the metadata provider via the ambient store; no MBQL 5 path yet^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.util :as qp.util]- [metabase.util :as u][metabase.util.i18n :refer [tru]][metabase.util.log :as log][metabase.util.malli :as mu]))@@ -67,20 +65,6 @@(throw (ex-info (tru "Querying this database requires the audit-app feature flag")query)))-(mu/defn remove-internal-keys :- ::lib.schema/query- "Pre-processing middleware. Strip internal query-processor keys from the incoming `query` so that they can only ever- be set by the query processor itself, not supplied by a client. Skipped while re-running pivot sub-queries, which- legitimately carry these keys."- [query :- ::lib.schema/query]- (cond-> query- (not qp.pipeline/*pivot?*) lib/prepare-after-deserialization))--(mu/defn record-referenced-card-ids :- ::lib.schema/query- "Pre-processing middleware. Record the source-card IDs referenced by `query` under the- `:query-permissions/referenced-card-ids` key."- [query :- ::lib.schema/query]- (u/assoc-dissoc query :query-permissions/referenced-card-ids (lib/all-source-card-ids-recursive query)))-(mu/defn check-query-permissions*"Check that User with `user-id` has permissions to run `query`, or throw an exception."[query :- ::qp.schema/any-query]diff --git a/src/metabase/query_processor/middleware/permissions/preprocess.clj b/src/metabase/query_processor/middleware/permissions/preprocess.cljnew file mode 100644index 00000000000..67c7cd2c9c7--- /dev/null+++ b/src/metabase/query_processor/middleware/permissions/preprocess.clj@@ -0,0 +1,28 @@+(ns metabase.query-processor.middleware.permissions.preprocess+ "Pre-processing middleware that prepares an incoming query for the permission checks that run later.++ Separate from [[metabase.query-processor.middleware.permissions]] because that namespace needs the+ `query-permissions` module to do the checking, and `query-permissions` in turn+ needs [[metabase.query-processor.preprocess]] to resolve the sources of a query."+ (:require+ [metabase.lib.core :as lib]+ [metabase.lib.schema :as lib.schema]+ [metabase.query-processor.pipeline :as qp.pipeline]+ [metabase.util :as u]+ [metabase.util.malli :as mu]))++(set! *warn-on-reflection* true)++(mu/defn remove-internal-keys :- ::lib.schema/query+ "Pre-processing middleware. Strip internal query-processor keys from the incoming `query` so that they can only ever+ be set by the query processor itself, not supplied by a client. Skipped while re-running pivot sub-queries, which+ legitimately carry these keys."+ [query :- ::lib.schema/query]+ (cond-> query+ (not qp.pipeline/*pivot?*) lib/prepare-after-deserialization))++(mu/defn record-referenced-card-ids :- ::lib.schema/query+ "Pre-processing middleware. Record the source-card IDs referenced by `query` under the+ `:query-permissions/referenced-card-ids` key."+ [query :- ::lib.schema/query]+ (u/assoc-dissoc query :query-permissions/referenced-card-ids (lib/all-source-card-ids-recursive query)))diff --git a/src/metabase/query_processor/preprocess.clj b/src/metabase/query_processor/preprocess.cljindex eaff3e0b7f7..fc91aae79eb 100644--- a/src/metabase/query_processor/preprocess.clj+++ b/src/metabase/query_processor/preprocess.clj@@ -30,7 +30,7 @@[metabase.query-processor.middleware.normalize-query :as normalize][metabase.query-processor.middleware.optimize-temporal-filters :as optimize-temporal-clauses][metabase.query-processor.middleware.parameters :as parameters]- [metabase.query-processor.middleware.permissions :as qp.perms]+ [metabase.query-processor.middleware.permissions.preprocess :as qp.perms.preprocess][metabase.query-processor.middleware.persistence :as qp.persistence][metabase.query-processor.middleware.prefetch-metadata :as prefetch-metadata][metabase.query-processor.middleware.reconcile-breakout-and-order-by-bucketing :as reconcile-bucketing]@@ -60,8 +60,8 @@All of these middlewares assume MBQL 5.";; ↓↓↓ PRE-PROCESSING ↓↓↓ happens from TOP TO BOTTOM[#'normalize/normalize-preprocessing-middleware- #'qp.perms/remove-internal-keys- #'qp.perms/record-referenced-card-ids+ #'qp.perms.preprocess/remove-internal-keys+ #'qp.perms.preprocess/record-referenced-card-ids#'qp.constraints/maybe-add-default-userland-constraints#'validate/validate-query#'prefetch-metadata/prefetch-metadatadiff --git a/src/metabase/query_processor/util/persisted_cache.clj b/src/metabase/query_processor/util/persisted_cache.cljindex 490b3f8524b..8e47f84070d 100644--- a/src/metabase/query_processor/util/persisted_cache.clj+++ b/src/metabase/query_processor/util/persisted_cache.clj@@ -4,6 +4,8 @@(:require[metabase.driver :as driver][metabase.driver.ddl.interface :as ddl.i]+ ;; We should not be using specific driver implementations+ [metabase.driver.sql.util :as sql.u][metabase.driver.util :as driver.u][metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.metadata :as lib.schema.metadata]@@ -32,16 +34,14 @@persistence is appropriate. Use [[can-substitute?]] for that check."[database-id :- ::lib.schema.id/database{:keys [table-name] :as _persisted-info} :- ::lib.schema.metadata/persisted-info]- (let [driver (or driver/*driver* (driver.u/database->driver database-id))- ;; We should not be using specific driver implementations- quote-name (requiring-resolve 'metabase.driver.sql.util/quote-name)]+ (let [driver (or driver/*driver* (driver.u/database->driver database-id))];; select * because we don't actually know the name of the fields when in the actual query. See #28902(format "select * from %s.%s"- (quote-name+ (sql.u/quote-namedriver:table(ddl.i/schema-name {:id database-id} (system/site-uuid)))- (quote-name+ (sql.u/quote-namedriver:tabletable-name))))diff --git a/src/metabase/search/impl.clj b/src/metabase/search/impl.cljindex 455149a23dd..331eb091d5d 100644--- a/src/metabase/search/impl.clj+++ b/src/metabase/search/impl.clj@@ -1,6 +1,7 @@(ns metabase.search.impl(:require[clojure.string :as str]+ [metabase.api.common :as api][metabase.collections.models.collection :as collection][metabase.collections.models.collection.root :as collection.root][metabase.models.interface :as mi]@@ -43,8 +44,8 @@on dynamic variables."{:style/indent 0}[current-user-id current-user-perms & body]- `(with-bindings {(requiring-resolve 'metabase.api.common/*current-user-id*) ~current-user-id- (requiring-resolve 'metabase.api.common/*current-user-permissions-set*) (atom ~current-user-perms)}+ `(binding [api/*current-user-id* ~current-user-id+ api/*current-user-permissions-set* (atom ~current-user-perms)]~@body))(defn- can-write? [{:keys [current-user-id current-user-perms]} instance]diff --git a/src/metabase/search/in_place/filter.clj b/src/metabase/search/in_place/filter.cljindex 0365e68946a..211b30a0fa7 100644--- a/src/metabase/search/in_place/filter.clj+++ b/src/metabase/search/in_place/filter.clj@@ -19,6 +19,7 @@[metabase.query-processor.parameters.dates :as params.dates][metabase.search.config :as search.config :refer [SearchableModel SearchContext]][metabase.search.filter :as search.filter]+ [metabase.search.in-place.search-model :as search-model][metabase.search.in-place.util :as search.util][metabase.search.permissions :as search.permissions][metabase.search.spec :as search.spec]@@ -74,8 +75,7 @@(when-let [query (:search-string search-context)](into[:or]- (for [column (->> (let [search-columns-fn (requiring-resolve 'metabase.search.in-place.legacy/searchable-columns)]- (search-columns-fn model search-native-query))+ (for [column (->> (search-model/searchable-columns model search-native-query)(map #(search.config/column-with-model-alias model %)))wildcarded-token (->> (search.util/normalize query)search.util/tokenize@@ -267,9 +267,6 @@(->> (get query join-type) (partition 2) (map first) (some #(= % table)) boolean));; We won't need this post-legacy as it defines the joins à la carte.-(defn- search-model->revision-model [model]- ((requiring-resolve 'metabase.search.in-place.legacy/search-model->revision-model) model))-(doseq [model ["dashboard" "card" "dataset" "metric"]](defmethod build-optional-filter-query [:last-edited-by model][_filter model query editor-ids]@@ -278,7 +275,7 @@(not (joined-with-table? query :join :revision))(-> (sql.helpers/join :revision [:= :revision.model_id (search.config/column-with-model-alias model :id)])(sql.helpers/where [:= :revision.most_recent true]- [:= :revision.model (search-model->revision-model model)]))+ [:= :revision.model (search-model/search-model->revision-model model)]))(= 1 (count editor-ids))(sql.helpers/where [:= :revision.user_id (first editor-ids)])@@ -293,7 +290,7 @@(not (joined-with-table? query :join :revision))(-> (sql.helpers/join :revision [:= :revision.model_id (search.config/column-with-model-alias model :id)])(sql.helpers/where [:= :revision.most_recent true]- [:= :revision.model (search-model->revision-model model)]))+ [:= :revision.model (search-model/search-model->revision-model model)]))true;; on UI we showed the the last edit info from revision.timestamp;; not the model.updated_at column@@ -362,7 +359,7 @@(-> (merge;; models support search-native-query if there are additional columns to search when the `search-native-query`;; argument is true- {:search-native-query (->> (dissoc (methods @(requiring-resolve 'metabase.search.in-place.legacy/searchable-columns)) :default)+ {:search-native-query (->> (dissoc (methods search-model/searchable-columns) :default)(filter (fn [[model f]](seq (set/difference (set (f model true)) (set (f model false))))))(map first)diff --git a/src/metabase/search/in_place/legacy.clj b/src/metabase/search/in_place/legacy.cljindex 817e2749e80..b31b002f77b 100644--- a/src/metabase/search/in_place/legacy.clj+++ b/src/metabase/search/in_place/legacy.clj@@ -14,6 +14,7 @@[metabase.search.filter :as search.filter][metabase.search.in-place.filter :as search.in-place.filter][metabase.search.in-place.scoring :as scoring]+ [metabase.search.in-place.search-model :as search-model][metabase.search.in-place.util :as search.util][metabase.search.permissions :as search.permissions][metabase.util.honey-sql-2 :as h2x]@@ -35,14 +36,6 @@(when (seq terms)[(str/join " " terms)]))-(defn search-model->revision-model- "Return the appropriate revision model given a search model."- [model]- (case model- "dataset" (recur "card")- "metric" (recur "card")- (str/capitalize model)))-(mu/defn- ->column-alias :- keyword?"Returns the column name. If the column is aliased, i.e. [`:original_name` `:aliased_name`], return the aliasedcolumn name"@@ -247,7 +240,7 @@(sql.helpers/left-join [:revision :r][:and [:= :r.model_id (search.config/column-with-model-alias model :id)][:= :r.most_recent true]- [:= :r.model (search-model->revision-model model)]])))+ [:= :r.model (search-model/search-model->revision-model model)]])))(mu/defn- with-moderated-status :- :map[query :- :map@@ -285,85 +278,6 @@{:arglists '([model search-context])}(fn [model _] model))-(defmulti searchable-columns- "The columns that can be searched for each model."- {:arglists '([model search-native-query])}- (fn [model _] model))--(defmethod searchable-columns :default- [_ _]- [:name])--(defmethod searchable-columns "action"- [_ search-native-query]- (cond-> [:name- :description]- search-native-query- (conj :dataset_query)))--(defmethod searchable-columns "card"- [_ search-native-query]- (cond-> [:name- :description]- search-native-query- (conj :dataset_query)))--(defmethod searchable-columns "dataset"- [_ search-native-query]- (searchable-columns "card" search-native-query))--(defmethod searchable-columns "measure"- [_ _]- [:name- :description])--(defmethod searchable-columns "metric"- [_ search-native-query]- (searchable-columns "card" search-native-query))--(defmethod searchable-columns "dashboard"- [_ _]- [:name- :description])--(defmethod searchable-columns "page"- [_ search-native-query]- (searchable-columns "dashboard" search-native-query))--(defmethod searchable-columns "database"- [_ _]- [:name- :description])--(defmethod searchable-columns "table"- [_ _]- [:name- :display_name- :description])--(defmethod searchable-columns "transform"- [_ search-native-query]- (cond-> [:name- :description]- search-native-query- (conj :source)))--(defmethod searchable-columns "indexed-entity"- [_ _]- [:name])--(defmethod searchable-columns "document"- [_ _]- [:name- :document])--;; mirrors the appdb spec's :search-terms [:name :description] (see-;; metabase.explorations.models.exploration)-(defmethod searchable-columns "exploration"- [_ _]- [:name- :description])-(def ^:private default-columns"Columns returned for all models."[:id :name :description :archived :created_at :updated_at])diff --git a/src/metabase/search/in_place/scoring.clj b/src/metabase/search/in_place/scoring.cljindex da965c3239b..0d55b12e562 100644--- a/src/metabase/search/in_place/scoring.clj+++ b/src/metabase/search/in_place/scoring.clj@@ -115,6 +115,7 @@[java-time.api :as t][metabase.premium-features.core :refer [defenterprise]][metabase.search.config :as search.config]+ [metabase.search.in-place.search-model :as search-model][metabase.search.in-place.util :as search.util][metabase.util :as u]))@@ -153,19 +154,12 @@{:is_match is-match:text (tokens->string text-tokens (not is-match))})))))-(let [resolved-fn (delay (requiring-resolve 'metabase.search.in-place.legacy/searchable-columns))]- (defn- searchable-columns-fn- "Get searchable columns for a model.- Lazily resolves the function from the legacy namespace to avoid circular dependencies."- [model search-native-query]- (@resolved-fn model search-native-query)))-(defn- text-scores-with"Scores a search result. Returns a vector of score maps, each containing `:weight`, `:score`, and other info aboutthe text match, if there is one. If there is no match, the score is 0."[search-native-query weighted-scorers query-tokens search-result];; TODO is pmap over search-result worth it?- (let [scores (for [column (searchable-columns-fn (:model search-result) search-native-query)+ (let [scores (for [column (search-model/searchable-columns (:model search-result) search-native-query){:keys [scorer name weight]:as _ws} weighted-scorers:let [matched-text (-> search-resultdiff --git a/src/metabase/search/in_place/search_model.clj b/src/metabase/search/in_place/search_model.cljnew file mode 100644index 00000000000..4acb8619411--- /dev/null+++ b/src/metabase/search/in_place/search_model.clj@@ -0,0 +1,95 @@+(ns metabase.search.in-place.search-model+ "Per-search-model facts shared by the in-place engine's namespaces.++ These live here rather than in [[metabase.search.in-place.legacy]] because+ [[metabase.search.in-place.filter]] and [[metabase.search.in-place.scoring]] both need them, and `legacy` reads+ those two namespaces."+ (:require+ [clojure.string :as str]))++(defn search-model->revision-model+ "Return the appropriate revision model given a search model."+ [model]+ (case model+ "dataset" (recur "card")+ "metric" (recur "card")+ (str/capitalize model)))++(defmulti searchable-columns+ "The columns that can be searched for each model."+ {:arglists '([model search-native-query])}+ (fn [model _] model))++(defmethod searchable-columns :default+ [_ _]+ [:name])++(defmethod searchable-columns "action"+ [_ search-native-query]+ (cond-> [:name+ :description]+ search-native-query+ (conj :dataset_query)))++(defmethod searchable-columns "card"+ [_ search-native-query]+ (cond-> [:name+ :description]+ search-native-query+ (conj :dataset_query)))++(defmethod searchable-columns "dataset"+ [_ search-native-query]+ (searchable-columns "card" search-native-query))++(defmethod searchable-columns "measure"+ [_ _]+ [:name+ :description])++(defmethod searchable-columns "metric"+ [_ search-native-query]+ (searchable-columns "card" search-native-query))++(defmethod searchable-columns "dashboard"+ [_ _]+ [:name+ :description])++(defmethod searchable-columns "page"+ [_ search-native-query]+ (searchable-columns "dashboard" search-native-query))++(defmethod searchable-columns "database"+ [_ _]+ [:name+ :description])++(defmethod searchable-columns "table"+ [_ _]+ [:name+ :display_name+ :description])++(defmethod searchable-columns "transform"+ [_ search-native-query]+ (cond-> [:name+ :description]+ search-native-query+ (conj :source)))++(defmethod searchable-columns "indexed-entity"+ [_ _]+ [:name])++(defmethod searchable-columns "document"+ [_ _]+ [:name+ :document])++;; mirrors the appdb spec's :search-terms [:name :description] (see+;; metabase.explorations.models.exploration)+(defmethod searchable-columns "exploration"+ [_ _]+ [:name+ :description])diff --git a/src/metabase/search/permissions.clj b/src/metabase/search/permissions.cljindex 122960d0899..5429f92d437 100644--- a/src/metabase/search/permissions.clj+++ b/src/metabase/search/permissions.clj@@ -1,5 +1,6 @@(ns metabase.search.permissions(:require+ [metabase.api.common :as api][metabase.collections.models.collection :as collection][metabase.models.interface :as mi][metabase.permissions.core :as perms]@@ -7,7 +8,7 @@[metabase.util.malli :as mu]))(defn- assert-current-user! [missing-param]- (assert @@(requiring-resolve 'metabase.api.common/*current-user*)+ (assert @api/*current-user*(format "metabase.api.common/*current-user* must be bound if %s is missing from search-ctx" missing-param)))(defn- impersonated-user? [{:keys [is-impersonated-user?] :as _search-ctx}]diff --git a/src/metabase/settings/models/setting.clj b/src/metabase/settings/models/setting.cljindex 6a3b6cd95f1..dfa4c81e512 100644--- a/src/metabase/settings/models/setting.clj+++ b/src/metabase/settings/models/setting.clj@@ -10,6 +10,7 @@[malli.core :as mc][medley.core :as m][metabase.api.common :as api]+ [metabase.app-db.core :as app-db][metabase.app-db.setting :as mdb.setting][metabase.config.core :as config][metabase.events.core :as events]@@ -535,14 +536,6 @@(binding [*disable-init* true](get setting-definition-or-name)))-(def ^:private db-is-set-up-var (atom nil))--(defn- db-is-set-up? []- ;; this should never be hit. it is just overly cautious against a NPE here. But no way this cannot resolve- (let [f (or @db-is-set-up-var- (reset! db-is-set-up-var (requiring-resolve 'metabase.app-db.core/db-is-set-up?)))]- (if f (f) false)))-(defn- db-or-cache-value*"Look up a single setting key in the DB or cache. Returns the raw (possibly empty) string, or nil."^String [setting-name-str]@@ -567,7 +560,7 @@^String [setting-definition-or-name](let [setting (resolve-setting setting-definition-or-name)];; cannot use db (and cache populated from db) if db is not set up- (when (and (db-is-set-up?) (allows-site-wide-values? setting))+ (when (and (app-db/db-is-set-up?) (allows-site-wide-values? setting))(or (not-empty (db-or-cache-value* (setting-name setting)))(when-let [deprecated-name (:deprecated-name setting)](when-let [v (not-empty (db-or-cache-value* (setting-name deprecated-name)))]@@ -591,7 +584,7 @@(defn- init! [setting-definition-or-name](let [{:keys [init] :as setting} (resolve-setting setting-definition-or-name)](when init- (when (not (db-is-set-up?))+ (when (not (app-db/db-is-set-up?))(throw (ex-info "Cannot initialize setting before the db is set up" {:setting setting})));; We do not need to interact with the restore-cache-lock as it is OK to race with it.(if-not (.tryLock init-lock 30 TimeUnit/SECONDS)diff --git a/src/metabase/sso/core.clj b/src/metabase/sso/core.cljindex cb82ef2df72..992f885ba24 100644--- a/src/metabase/sso/core.clj+++ b/src/metabase/sso/core.clj@@ -4,6 +4,7 @@[metabase.sso.google][metabase.sso.ldap][metabase.sso.ldap.default-implementation]+ [metabase.sso.ldap.settings][metabase.sso.oidc.check][metabase.sso.oidc.state][metabase.sso.settings]@@ -26,10 +27,11 @@[metabase.sso.settingsgoogle-auth-client-idgoogle-auth-enabled- ldap-enabledsend-new-sso-user-admin-email?sso-enabled?- sso-source-enabled?])+ sso-source-enabled?]+ [metabase.sso.ldap.settings+ ldap-enabled]);; the re-exported var carries the docstring; kondo can't see through import-def#_{:clj-kondo/ignore [:missing-docstring]}diff --git a/src/metabase/sso/init.clj b/src/metabase/sso/init.cljindex a0718f33e4c..fee8efa4235 100644--- a/src/metabase/sso/init.clj+++ b/src/metabase/sso/init.clj@@ -1,5 +1,6 @@(ns metabase.sso.init(:require+ [metabase.sso.ldap.settings][metabase.sso.providers.google][metabase.sso.providers.ldap][metabase.sso.providers.oidc]diff --git a/src/metabase/sso/ldap/settings.clj b/src/metabase/sso/ldap/settings.cljnew file mode 100644index 00000000000..89196913dcb--- /dev/null+++ b/src/metabase/sso/ldap/settings.clj@@ -0,0 +1,29 @@+(ns metabase.sso.ldap.settings+ "The `ldap-enabled` setting, which refuses to turn LDAP on until the configured server actually answers.++ It lives here rather than alongside the other SSO settings because that check runs+ [[metabase.sso.ldap/test-current-ldap-details]], while [[metabase.sso.ldap]] reads the LDAP connection settings+ from [[metabase.sso.settings]] -- defining it there would make those two namespaces require each other.++ Nothing refers to this namespace by name; the setting reaches the settings API only because+ [[metabase.sso.init]] loads it."+ (:require+ [metabase.settings.core :as setting :refer [defsetting]]+ [metabase.sso.ldap :as ldap]+ [metabase.util.i18n :refer [deferred-tru tru]]))++(defsetting ldap-enabled+ (deferred-tru "Is LDAP currently enabled?")+ :type :boolean+ :visibility :public+ :setter (fn [new-value]+ (let [new-value (boolean new-value)]+ (when new-value+ ;; Test the LDAP settings before enabling+ (let [result (ldap/test-current-ldap-details)]+ (when-not (= :SUCCESS (:status result))+ (throw (ex-info (tru "Unable to connect to LDAP server with current settings")+ (ldap/humanize-error-messages result))))))+ (setting/set-value-of-type! :boolean :ldap-enabled new-value)))+ :default false+ :audit :getter)diff --git a/src/metabase/sso/providers/slack_connect.clj b/src/metabase/sso/providers/slack_connect.cljindex 937cdf73a00..e9bb3ed563b 100644--- a/src/metabase/sso/providers/slack_connect.clj+++ b/src/metabase/sso/providers/slack_connect.clj@@ -6,6 +6,7 @@[metabase.server.settings :as server.settings][metabase.sso.db :as sso.db][metabase.sso.settings :as sso-settings]+ [metabase.system.core :as system][metabase.util.i18n :refer [tru]][methodical.core :as methodical]))@@ -82,7 +83,7 @@[redirect-url](try(let [redirect (some-> redirect-url (java.net.URI.))- our-host (some-> ((requiring-resolve 'metabase.system.core/site-url)) (java.net.URI.) (.getHost))]+ our-host (some-> (system/site-url) (java.net.URI.) (.getHost))](when-not (or (nil? redirect-url)(and (nil? (.getHost redirect))(nil? (.getScheme redirect)))diff --git a/src/metabase/sso/settings.clj b/src/metabase/sso/settings.cljindex 8e7f5c6a7a8..216462a2415 100644--- a/src/metabase/sso/settings.clj+++ b/src/metabase/sso/settings.clj@@ -128,22 +128,6 @@(ldap-user-base)))):doc false)-(defsetting ldap-enabled- (deferred-tru "Is LDAP currently enabled?")- :type :boolean- :visibility :public- :setter (fn [new-value]- (let [new-value (boolean new-value)]- (when new-value- ;; Test the LDAP settings before enabling- (let [result ((requiring-resolve 'metabase.sso.ldap/test-current-ldap-details))]- (when-not (= :SUCCESS (:status result))- (throw (ex-info (tru "Unable to connect to LDAP server with current settings")- ((requiring-resolve 'metabase.sso.ldap/humanize-error-messages) result))))))- (setting/set-value-of-type! :boolean :ldap-enabled new-value)))- :default false- :audit :getter)-(defsetting ldap-timeout-seconds(deferred-tru "Maximum time, in seconds, to wait for LDAP server before falling back to local authentication"):type :double@@ -296,7 +280,9 @@"Any SSO provider is configured and enabled"[](or (google-auth-enabled)- (ldap-enabled)+ ;; read by keyword the way `ee-sso-configured?` above does: `ldap-enabled` is defined in+ ;; [[metabase.sso.ldap.settings]], which this namespace cannot require+ (setting/get :ldap-enabled)(ee-sso-configured?)))(defn sso-source-enabled?@@ -307,7 +293,7 @@(boolean(case (keyword sso-source):google (google-auth-enabled)- :ldap (ldap-enabled)+ :ldap (setting/get :ldap-enabled);; Enterprise SSO providers: setting/get respects the :feature flag on each;; setting — returning the default (false) when the feature is unlicensed (e.g.,;; after license downgrade), so users aren't locked out of password reset.diff --git a/src/metabase/sync/fetch_metadata.clj b/src/metabase/sync/fetch_metadata.cljindex 605e9c960f0..afa7c6e7194 100644--- a/src/metabase/sync/fetch_metadata.clj+++ b/src/metabase/sync/fetch_metadata.clj@@ -5,6 +5,7 @@(:require[clojure.set :as set][metabase.driver :as driver]+ [metabase.driver.sql-jdbc.sync :as sql-jdbc.sync][metabase.driver.util :as driver.u][metabase.lib-be.core :as lib-be][metabase.sync.db :as sync.db]@@ -37,7 +38,7 @@(let [driver (driver.u/database->driver database)](cond-> fields(driver.u/supports? driver :nested-field-columns database)- (set/union ((requiring-resolve 'metabase.driver.sql-jdbc.sync/describe-nested-field-columns) driver database table)))))+ (set/union (sql-jdbc.sync/describe-nested-field-columns driver database table)))))(mu/defn table-fields-metadata :- [:set i/TableMetadataField]"Fetch metadata about Fields belonging to a given `table` directly from an external database by calling its driver'sdiff --git a/src/metabase/sync/task/sync_databases.clj b/src/metabase/sync/task/sync_databases.cljindex 37d70e46be5..86b95ec252b 100644--- a/src/metabase/sync/task/sync_databases.clj+++ b/src/metabase/sync/task/sync_databases.clj@@ -6,8 +6,6 @@(:require[clojurewerkz.quartzite.conversion :as qc][clojurewerkz.quartzite.jobs :as jobs]- [clojurewerkz.quartzite.schedule.cron :as cron]- [clojurewerkz.quartzite.triggers :as triggers][java-time.api :as t][metabase.audit-app.core :as audit][metabase.config.core :as config]@@ -21,10 +19,10 @@[metabase.sync.field-values :as sync.field-values][metabase.sync.schedules :as sync.schedules][metabase.sync.sync-metadata :as sync-metadata]+ [metabase.sync.task.sync-databases-trigger :as sync-databases-trigger][metabase.task.core :as task][metabase.tracing.core :as tracing][metabase.util :as u]- [metabase.util.cron :as u.cron][metabase.util.log :as log][metabase.util.malli :as mu][metabase.util.malli.registry :as mr]@@ -32,10 +30,7 @@[metabase.warehouses.core :as warehouses])(:import(org.quartz- CronTrigger- JobDetail- JobKey- TriggerKey)))+ JobDetail)))(set! *warn-on-reflection* true)@@ -43,8 +38,6 @@;;; | JOB LOGIC |;;; +----------------------------------------------------------------------------------------------------------------+-(declare unschedule-tasks-for-db!)-(mu/defn- job-context->database-id :- [:maybe ::lib.schema.id/database]"Get the Database ID referred to in `job-context`."[job-context]@@ -78,7 +71,7 @@(log/infof "Starting sync task for Database %d." database-id)(when-let [database (or (sync.db/database database-id)(do- (unschedule-tasks-for-db! (mi/instance :model/Database {:id database-id}))+ (sync-databases-trigger/unschedule-tasks-for-db! (mi/instance :model/Database {:id database-id}))(log/warnf "Cannot sync Database %d: Database does not exist." database-id)))](if-let [ex (try;; it's okay to allow testing H2 connections during sync. We only want to disallow you from testing them for the@@ -143,7 +136,7 @@(log/infof "Update Field values task triggered for Database %d." database-id)(when-let [database (or (sync.db/database database-id)(do- (unschedule-tasks-for-db! (mi/instance :model/Database {:id database-id}))+ (sync-databases-trigger/unschedule-tasks-for-db! (mi/instance :model/Database {:id database-id}))(log/warnf "Cannot update Field values for Database %d: Database does not exist." database-id)))](if (:is_full_sync database)(sync.field-values/update-field-values! database)@@ -155,188 +148,25 @@(update-field-values! job-context));;; +----------------------------------------------------------------------------------------------------------------+-;;; | TASK INFO AND GETTER FUNCTIONS |+;;; | JOB DEFINITIONS |;;; +----------------------------------------------------------------------------------------------------------------+(mr/def ::class[:fn {:error/message "a Class"} class?])-(def ^:private TaskInfo- "One-off schema for information about the various sync tasks we run for a DB."- [:map- [:key :keyword]- [:db-schedule-column :keyword]- [:job-class ::class]- [:name :string]])--(def ^:private sync-analyze-task-info- {:key :sync-and-analyze- :db-schedule-column :metadata_sync_schedule- :job-class SyncAndAnalyzeDatabase- :name "Sync and Analyze"})--(assert (mr/validate TaskInfo sync-analyze-task-info))--(def ^:private field-values-task-info- {:key :update-field-values- :db-schedule-column :cache_field_values_schedule- :job-class UpdateFieldValues- :name "Scan Field Values"})--(assert (mr/validate TaskInfo field-values-task-info))--(def ^:private all-tasks- [sync-analyze-task-info field-values-task-info])--;; These getter functions are not strictly necessary but are provided primarily so we can get some extra validation by-;; using them--(mu/defn- job-key :- (ms/InstanceOfClass JobKey)- "Return an appropriate string key for the job described by `task-info` for `database-or-id`."- ^JobKey [task-info :- TaskInfo]- (jobs/key (format "metabase.task.%s.job" (name (:key task-info)))))--(mu/defn- trigger-key :- (ms/InstanceOfClass TriggerKey)- "Return an appropriate string key for the trigger for `task-info` and `database-or-id`."- ^TriggerKey [database :- (ms/InstanceOf :model/Database)- task-info :- TaskInfo]- (triggers/key (format "metabase.task.%s.trigger.%d" (name (:key task-info)) (u/the-id database))))--(mu/defn- cron-schedule :- [:maybe u.cron/CronScheduleString]- "Fetch the appropriate cron schedule string for `database` and `task-info`."- [database :- (ms/InstanceOf :model/Database)- task-info :- TaskInfo]- (get database (:db-schedule-column task-info)))--(mu/defn- job-class :- ::class- "Get the Job class for `task-info`."- [task-info :- TaskInfo]- (:job-class task-info))--(mu/defn- trigger-description :- :string- "Return an appropriate description string for a job/trigger for Database described by `task-info`."- [database :- (ms/InstanceOf :model/Database)- task-info :- TaskInfo]- (format "%s Database %d" (name (:key task-info)) (u/the-id database)))--(mu/defn- job-description :- :string- "Return an appropriate description string for a job"- [task-info :- TaskInfo]- (format "%s for all databases" (name (:key task-info))))--;;; +----------------------------------------------------------------------------------------------------------------+-;;; | DELETING TASKS FOR A DB |-;;; +----------------------------------------------------------------------------------------------------------------+--(mu/defn- delete-trigger!- "Cancel a single sync trigger for `database-or-id` and `task-info`."- [database :- (ms/InstanceOf :model/Database)- task-info :- TaskInfo]- (let [trigger-key (trigger-key database task-info)]- (log/debug (u/format-color 'red- (format "Unscheduling task for Database %d: trigger: %s" (u/the-id database) (.getName trigger-key))))- (task/delete-trigger! trigger-key)))--(mu/defn unschedule-tasks-for-db!- "Cancel *all* scheduled sync and FieldValues caching tasks for `database-or-id`."- [database :- (ms/InstanceOf :model/Database)]- (doseq [task all-tasks]- (delete-trigger! database task)))--;;; +----------------------------------------------------------------------------------------------------------------+-;;; | (RE)SCHEDULING TASKS FOR A DB |-;;; +----------------------------------------------------------------------------------------------------------------+-(mu/defn- job :- (ms/InstanceOfClass JobDetail)"Build a durable Quartz Job for `task-info`. Durable in Quartz allows the job to exist even if there are no triggersfor it."- ^JobDetail [task-info :- TaskInfo]+ ^JobDetail [task-info :- sync-databases-trigger/TaskInfo+ job-class :- ::class](jobs/build- (jobs/with-description (job-description task-info))- (jobs/of-type (job-class task-info))- (jobs/with-identity (job-key task-info))+ (jobs/with-description (sync-databases-trigger/job-description task-info))+ (jobs/of-type job-class)+ (jobs/with-identity (sync-databases-trigger/job-key task-info))(jobs/store-durably)))-(def ^:private sync-analyze-job (job sync-analyze-task-info))-(def ^:private field-values-job (job field-values-task-info))--(mu/defn- trigger :- [:maybe (ms/InstanceOfClass CronTrigger)]- "Build a Quartz Trigger for `database` and `task-info` if a schedule exists."- ^CronTrigger [database :- (ms/InstanceOf :model/Database)- task-info :- TaskInfo]- (when-let [task-schedule (cron-schedule database task-info)]- (triggers/build- (triggers/with-description (trigger-description database task-info))- (triggers/with-identity (trigger-key database task-info))- (triggers/using-job-data {"db-id" (u/the-id database)})- (triggers/for-job (job-key task-info))- (triggers/start-now)- (triggers/with-schedule- (cron/schedule- (cron/cron-schedule task-schedule)- ;; if we miss a sync for one reason or another (such as system being down) do not try to run the sync again.- ;; Just wait until the next sync cycle.- ;;- ;; See https://www.nurkiewicz.com/2012/04/quartz-scheduler-misfire-instructions.html for more info- (cron/with-misfire-handling-instruction-do-nothing))))))--(defn- update-db-trigger-if-needed!- "Replace or remove the existing trigger if the schedule changes, do nothing if schedule is the same."- [database task-info]- (let [job (task/job-info (job-key task-info))- new-trigger (trigger database task-info)- ;; there should be only one schedule per task per DB- existing-trigger-with-same-schedule (when new-trigger- (let [trigger-key (.. new-trigger getKey getName)- task-schedule (cron-schedule database task-info)]- (some #(when (and (= (:key %) trigger-key)- (= (:schedule %) task-schedule))- %)- (:triggers job))))]- (cond- ;; no new schedule- ;; delete the existing trigger- (nil? new-trigger)- (do- (log/infof "Trigger for \"%s\" of Database %d has been removed. It will no longer run on a schedule."- (:name task-info)- (u/the-id database))- (delete-trigger! database task-info))-- ;; need to recreate the new trigger- (and (some? new-trigger)- (nil? existing-trigger-with-same-schedule))- (do- (if (delete-trigger! database task-info)- (log/infof "Trigger for \"%s\" of Database %d has been updated. The new schedule is: \"%s\""- (:name task-info)- (u/the-id database)- (cron-schedule database task-info))- (log/infof "A trigger for \"%s\" of Database %d has been enabled with schedule: \"%s\""- (:name task-info)- (u/the-id database)- (cron-schedule database task-info)))- (task/add-trigger! new-trigger))-- ;; don't need to do anything as the existing trigger matches the new schedule- :else- nil)))--;; called [[from metabase.warehouses.models.database/schedule-tasks!]] from the post-insert and the pre-update-(mu/defn check-and-schedule-tasks-for-db!- "Schedule a new Quartz job for `database` and `task-info` if it doesn't already exist or is incorrect."- [database :- (ms/InstanceOf :model/Database)]- (doseq [task all-tasks]- (cond- (:is_stub database)- (log/info (u/format-color :red "Not scheduling sync task for stub database"))-- (and (= audit/audit-db-id (:id database))- (= task sync-analyze-task-info))- (log/info (u/format-color :red "Not scheduling sync task for audit database"))-- :else- (update-db-trigger-if-needed! database task))))+(def ^:private sync-analyze-job (job sync-databases-trigger/sync-analyze-task-info SyncAndAnalyzeDatabase))+(def ^:private field-values-job (job sync-databases-trigger/field-values-task-info UpdateFieldValues));;; +----------------------------------------------------------------------------------------------------------------+;;; | TASK INITIALIZATION |diff --git a/src/metabase/sync/task/sync_databases_trigger.clj b/src/metabase/sync/task/sync_databases_trigger.cljnew file mode 100644index 00000000000..369da9fbe2d--- /dev/null+++ b/src/metabase/sync/task/sync_databases_trigger.clj@@ -0,0 +1,186 @@+(ns metabase.sync.task.sync-databases-trigger+ "Quartz trigger lifecycle for the per-Database sync-and-analyze and field-values tasks.++ Separate from [[metabase.sync.task.sync-databases]] so that :model/Database lifecycle hooks can (un)schedule+ triggers: the job bodies over there pull in the sync machinery, which reads the Database model."+ (:require+ [clojurewerkz.quartzite.jobs :as jobs]+ [clojurewerkz.quartzite.schedule.cron :as cron]+ [clojurewerkz.quartzite.triggers :as triggers]+ [metabase.audit-app.core :as audit]+ [metabase.task.core :as task]+ [metabase.util :as u]+ [metabase.util.cron :as u.cron]+ [metabase.util.log :as log]+ [metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr]+ [metabase.util.malli.schema :as ms])+ (:import+ (org.quartz+ CronTrigger+ JobKey+ TriggerKey)))++(set! *warn-on-reflection* true)++;;; +----------------------------------------------------------------------------------------------------------------++;;; | TASK INFO AND GETTER FUNCTIONS |+;;; +----------------------------------------------------------------------------------------------------------------+++(def TaskInfo+ "One-off schema for information about the various sync tasks we run for a DB."+ [:map+ [:key :keyword]+ [:db-schedule-column :keyword]+ [:name :string]])++(def sync-analyze-task-info+ "Task info for the sync-and-analyze task."+ {:key :sync-and-analyze+ :db-schedule-column :metadata_sync_schedule+ :name "Sync and Analyze"})++(assert (mr/validate TaskInfo sync-analyze-task-info))++(def field-values-task-info+ "Task info for the Field values caching task."+ {:key :update-field-values+ :db-schedule-column :cache_field_values_schedule+ :name "Scan Field Values"})++(assert (mr/validate TaskInfo field-values-task-info))++(def ^:private all-tasks+ [sync-analyze-task-info field-values-task-info])++;; These getter functions are not strictly necessary but are provided primarily so we can get some extra validation by+;; using them++(mu/defn job-key :- (ms/InstanceOfClass JobKey)+ "Return an appropriate string key for the job described by `task-info` for `database-or-id`."+ ^JobKey [task-info :- TaskInfo]+ (jobs/key (format "metabase.task.%s.job" (name (:key task-info)))))++(mu/defn- trigger-key :- (ms/InstanceOfClass TriggerKey)+ "Return an appropriate string key for the trigger for `task-info` and `database-or-id`."+ ^TriggerKey [database :- (ms/InstanceOf :model/Database)+ task-info :- TaskInfo]+ (triggers/key (format "metabase.task.%s.trigger.%d" (name (:key task-info)) (u/the-id database))))++(mu/defn- cron-schedule :- [:maybe u.cron/CronScheduleString]+ "Fetch the appropriate cron schedule string for `database` and `task-info`."+ [database :- (ms/InstanceOf :model/Database)+ task-info :- TaskInfo]+ (get database (:db-schedule-column task-info)))++(mu/defn- trigger-description :- :string+ "Return an appropriate description string for a job/trigger for Database described by `task-info`."+ [database :- (ms/InstanceOf :model/Database)+ task-info :- TaskInfo]+ (format "%s Database %d" (name (:key task-info)) (u/the-id database)))++(mu/defn job-description :- :string+ "Return an appropriate description string for a job"+ [task-info :- TaskInfo]+ (format "%s for all databases" (name (:key task-info))))++;;; +----------------------------------------------------------------------------------------------------------------++;;; | DELETING TASKS FOR A DB |+;;; +----------------------------------------------------------------------------------------------------------------+++(mu/defn- delete-trigger!+ "Cancel a single sync trigger for `database-or-id` and `task-info`."+ [database :- (ms/InstanceOf :model/Database)+ task-info :- TaskInfo]+ (let [trigger-key (trigger-key database task-info)]+ (log/debug (u/format-color 'red+ (format "Unscheduling task for Database %d: trigger: %s" (u/the-id database) (.getName trigger-key))))+ (task/delete-trigger! trigger-key)))++(mu/defn unschedule-tasks-for-db!+ "Cancel *all* scheduled sync and FieldValues caching tasks for `database-or-id`."+ [database :- (ms/InstanceOf :model/Database)]+ (doseq [task all-tasks]+ (delete-trigger! database task)))++;;; +----------------------------------------------------------------------------------------------------------------++;;; | (RE)SCHEDULING TASKS FOR A DB |+;;; +----------------------------------------------------------------------------------------------------------------+++(mu/defn- trigger :- [:maybe (ms/InstanceOfClass CronTrigger)]+ "Build a Quartz Trigger for `database` and `task-info` if a schedule exists."+ ^CronTrigger [database :- (ms/InstanceOf :model/Database)+ task-info :- TaskInfo]+ (when-let [task-schedule (cron-schedule database task-info)]+ (triggers/build+ (triggers/with-description (trigger-description database task-info))+ (triggers/with-identity (trigger-key database task-info))+ (triggers/using-job-data {"db-id" (u/the-id database)})+ (triggers/for-job (job-key task-info))+ (triggers/start-now)+ (triggers/with-schedule+ (cron/schedule+ (cron/cron-schedule task-schedule)+ ;; if we miss a sync for one reason or another (such as system being down) do not try to run the sync again.+ ;; Just wait until the next sync cycle.+ ;;+ ;; See https://www.nurkiewicz.com/2012/04/quartz-scheduler-misfire-instructions.html for more info+ (cron/with-misfire-handling-instruction-do-nothing))))))++(defn- update-db-trigger-if-needed!+ "Replace or remove the existing trigger if the schedule changes, do nothing if schedule is the same."+ [database task-info]+ (let [job (task/job-info (job-key task-info))+ new-trigger (trigger database task-info)+ ;; there should be only one schedule per task per DB+ existing-trigger-with-same-schedule (when new-trigger+ (let [trigger-key (.. new-trigger getKey getName)+ task-schedule (cron-schedule database task-info)]+ (some #(when (and (= (:key %) trigger-key)+ (= (:schedule %) task-schedule))+ %)+ (:triggers job))))]+ (cond+ ;; no new schedule+ ;; delete the existing trigger+ (nil? new-trigger)+ (do+ (log/infof "Trigger for \"%s\" of Database %d has been removed. It will no longer run on a schedule."+ (:name task-info)+ (u/the-id database))+ (delete-trigger! database task-info))++ ;; need to recreate the new trigger+ (and (some? new-trigger)+ (nil? existing-trigger-with-same-schedule))+ (do+ (if (delete-trigger! database task-info)+ (log/infof "Trigger for \"%s\" of Database %d has been updated. The new schedule is: \"%s\""+ (:name task-info)+ (u/the-id database)+ (cron-schedule database task-info))+ (log/infof "A trigger for \"%s\" of Database %d has been enabled with schedule: \"%s\""+ (:name task-info)+ (u/the-id database)+ (cron-schedule database task-info)))+ (task/add-trigger! new-trigger))++ ;; don't need to do anything as the existing trigger matches the new schedule+ :else+ nil)))++;; called [[from metabase.warehouses.models.database/schedule-tasks!]] from the post-insert and the pre-update+(mu/defn check-and-schedule-tasks-for-db!+ "Schedule a new Quartz job for `database` and `task-info` if it doesn't already exist or is incorrect."+ [database :- (ms/InstanceOf :model/Database)]+ (doseq [task all-tasks]+ (cond+ (:is_stub database)+ (log/info (u/format-color :red "Not scheduling sync task for stub database"))++ (and (= audit/audit-db-id (:id database))+ (= task sync-analyze-task-info))+ (log/info (u/format-color :red "Not scheduling sync task for audit database"))++ :else+ (update-db-trigger-if-needed! database task))))diff --git a/src/metabase/task/QUARTZ.md b/src/metabase/task/QUARTZ.mdindex 9562a9bfe41..dc7120485ae 100644--- a/src/metabase/task/QUARTZ.md+++ b/src/metabase/task/QUARTZ.md@@ -84,7 +84,7 @@ These tasks run on a recurring schedule defined by a cron expression.Notice in the example above that we set the timezone explicitly with `(cron/in-time-zone "UTC")`. By default, Quartz schedules are interpreted in the JVM's default timezone, but in Metabase we usually set the timezone to `report-timezone` .-**Important Note:** If your tasks are timezone sensitive, you'll need to rebuild the triggers when `driver/report-timezone` changes. See [[metabase.notification.task.send/update-send-notification-triggers-timezone!]] as an example of how we handle rescheduling.+**Important Note:** If your tasks are timezone sensitive, you'll need to rebuild the triggers when `driver/report-timezone` changes. See [[metabase.notification.task.send-trigger/update-send-notification-triggers-timezone!]] as an example of how we handle rescheduling.### 3. Tasks that run once like a migrationdiff --git a/src/metabase/task/bootstrap.clj b/src/metabase/task/bootstrap.cljindex 1cdfa48f00d..4cec974e70d 100644--- a/src/metabase/task/bootstrap.clj+++ b/src/metabase/task/bootstrap.clj@@ -6,12 +6,6 @@(set! *warn-on-reflection* true)-;; Custom `ConnectionProvider` implementation that uses a dedicated connection pool for the application DB to provide-;; connections.--(defn- quartz-data-source ^javax.sql.DataSource []- ((requiring-resolve 'metabase.app-db.core/quartz-data-source)))-;; Optional interceptor for wrapping JDBC connections before Quartz uses them.;; Set by task.tracing to add SQL-level tracing. nil means no interception.(defonce ^:private connection-interceptor (atom nil))@@ -22,28 +16,12 @@[f](reset! connection-interceptor f))-(defrecord ^:private ConnectionProvider []- org.quartz.utils.ConnectionProvider- (initialize [_])- (getConnection [_]- ;; get a connection from the dedicated Quartz connection pool. Quartz will close it (i.e., return it to the pool)- ;; when it's done.- ;;- ;; very important! Fetch a new connection from the connection pool rather than reusing a Connection already bound- ;; to the calling thread (e.g. toucan2's *current-connectable*) -- Quartz manages the connection's whole- ;; lifecycle (setAutoCommit/commit/rollback/close), and its cluster locking relies on commit/rollback to release- ;; row locks on the QRTZ_LOCKS table, so it must never share a connection with an outer transaction.- ;;- ;; the pool is separate from the main application DB pool so that a Quartz operation triggered by a thread inside- ;; a `with-transaction` block can't deadlock when application code has saturated the main pool.- (let [conn (.getConnection (quartz-data-source))]- (if-let [interceptor @connection-interceptor]- (interceptor conn)- conn)))- (shutdown [_]))--(when-not *compile-files*- (System/setProperty "org.quartz.dataSource.db.connectionProvider.class" (.getName ConnectionProvider)))+(defn intercept-connection+ "Wrap `conn` with the interceptor set by [[set-connection-interceptor!]], if any."+ [conn]+ (if-let [interceptor @connection-interceptor]+ (interceptor conn)+ conn))(defn- load-class ^Class [^String class-name](Class/forName class-name true (classloader/the-classloader)))diff --git a/src/metabase/tracing/quartz.clj b/src/metabase/tracing/quartz.cljindex 4f333a3f640..e1d93110d4c 100644--- a/src/metabase/tracing/quartz.clj+++ b/src/metabase/tracing/quartz.clj@@ -136,7 +136,7 @@(invoke-or-unwrap method conn args)))))))(defn- connection-interceptor- "Connection wrapping function installed into bootstrap's ConnectionProvider.+ "Connection wrapping function installed into the app-db's Quartz ConnectionProvider.Checks `:quartz` group at call time — wraps only when enabled."[^Connection conn](if (tracing/group-enabled? :quartz)diff --git a/src/metabase/warehouse_schema/models/field_values.clj b/src/metabase/warehouse_schema/models/field_values.cljindex 84dfeec92be..1e61496b200 100644--- a/src/metabase/warehouse_schema/models/field_values.clj+++ b/src/metabase/warehouse_schema/models/field_values.clj@@ -42,6 +42,7 @@[metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms][metabase.warehouse-schema.db :as warehouse-schema.db]+ [metabase.warehouse-schema.metadata-from-qp :as metadata-from-qp][methodical.core :as methodical][toucan2.core :as t2]))@@ -387,7 +388,7 @@(let [field (cond-> field;; a caller may hand us a Field hydrated with its Table, which is not part of column metadata(t2/model field) (-> (dissoc :table) (lib-be/instance->metadata :metadata/column)))- result ((requiring-resolve 'metabase.warehouse-schema.metadata-from-qp/table-query)+ result (metadata-from-qp/table-query(:table-id field)(fn [query](-> querydiff --git a/src/metabase/warehouses/models/database.clj b/src/metabase/warehouses/models/database.cljindex bf79193e94b..83d4c463837 100644--- a/src/metabase/warehouses/models/database.clj+++ b/src/metabase/warehouses/models/database.clj@@ -16,11 +16,12 @@[metabase.models.serialization :as serdes][metabase.permissions.core :as perms][metabase.premium-features.core :as premium-features :refer [defenterprise]]- ;; Trying to use metabase.search would cause a circular reference ;_;+ [metabase.search.core :as search][metabase.search.spec :as search.spec][metabase.secrets.core :as secret][metabase.settings.core :as setting][metabase.sync.schedules :as sync.schedules]+ [metabase.sync.task.sync-databases-trigger :as sync-databases-trigger][metabase.util :as u][metabase.util.honey-sql-2 :as h2x][metabase.util.i18n :refer [trs tru]]@@ -238,9 +239,8 @@"(Re)schedule sync operation tasks for `database`. (Existing scheduled tasks will be deleted first.)"[database](try- ;; this is done this way to avoid circular dependencies(when (should-auto-sync? database)- ((requiring-resolve 'metabase.sync.task.sync-databases/check-and-schedule-tasks-for-db!) database))+ (sync-databases-trigger/check-and-schedule-tasks-for-db! database))(catch Throwable e(log/errorf "Error scheduling tasks for DB: %s" (ex-message e)))))@@ -362,7 +362,7 @@"Unschedule any currently pending sync operation tasks for `database`."[database](try- ((requiring-resolve 'metabase.sync.task.sync-databases/unschedule-tasks-for-db!) database)+ (sync-databases-trigger/unschedule-tasks-for-db! database)(catch Throwable e(log/errorf "Error unscheduling tasks for DB: %s" (ex-message e)))))@@ -447,8 +447,7 @@(warehouses.db/delete-cards-for-database-returning-ids-reducible id)(warehouses.db/card-ids-for-database-reducible id)))(run! (fn [batch]- ;; damn circular deps- ((requiring-resolve 'metabase.search.core/delete!) :model/Card (map (comp str :id) batch)))))+ (search/delete! :model/Card (map (comp str :id) batch)))))(when (not= :postgres (mdb/db-type))(warehouses.db/delete-cards-for-database! id))(trydiff --git a/test/metabase/app_db/custom_migrations_test.clj b/test/metabase/app_db/custom_migrations_test.cljindex cb14136ed69..2e72908d3a9 100644--- a/test/metabase/app_db/custom_migrations_test.clj+++ b/test/metabase/app_db/custom_migrations_test.clj@@ -31,6 +31,7 @@[metabase.permissions.models.permissions-group :as perms-group][metabase.pulse.models.pulse-channel-test :as pulse-channel-test][metabase.pulse.task.send-pulses :as task.send-pulses]+ [metabase.pulse.task.send-pulses-trigger :as task.send-pulses-trigger][metabase.search.ingestion :as search.ingestion][metabase.settings.models.setting :as setting][metabase.sync.task.sync-databases-test :as task.sync-databases-test]@@ -1809,7 +1810,7 @@pulse-id (:id (new-instance-with-default :pulse {:creator_id user-id}))pc (new-instance-with-default :pulse_channel {:pulse_id pulse-id})];; trigger this so we schedule a trigger for send-pulse- (task.send-pulses/update-send-pulse-trigger-if-needed! pulse-id pc :add-pc-ids #{(:id pc)})+ (task.send-pulses-trigger/update-send-pulse-trigger-if-needed! pulse-id pc :add-pc-ids #{(:id pc)})(testing "sanity check that we have a send pulse trigger and 2 jobs"(is (= 1 (count (pulse-channel-test/send-pulse-triggers pulse-id))))(is (= #{"metabase.task.send-pulses.send-pulse.job"diff --git a/test/metabase/task/bootstrap_test.clj b/test/metabase/app_db/quartz_test.cljsimilarity index 94%rename from test/metabase/task/bootstrap_test.cljrename to test/metabase/app_db/quartz_test.cljindex 3441118412b..e129db54bfa 100644--- a/test/metabase/task/bootstrap_test.clj+++ b/test/metabase/app_db/quartz_test.clj@@ -1,10 +1,10 @@-(ns metabase.task.bootstrap-test+(ns metabase.app-db.quartz-test(:require[clojure.test :refer :all][metabase.app-db.connection :as mdb.connection][metabase.app-db.connection-pool-setup :as mdb.connection-pool-setup][metabase.app-db.data-source :as mdb.data-source]- [metabase.task.bootstrap :as task.bootstrap]+ [metabase.app-db.quartz :as mdb.quartz][toucan2.connection :as t2.conn])(:import(com.mchange.v2.c3p0 DataSources)@@ -23,7 +23,7 @@(binding [mdb.connection/*application-db* app-db](with-open [^Connection outer (.getConnection data-source)](binding [t2.conn/*current-connectable* outer]- (let [^ConnectionProvider provider (task.bootstrap/->ConnectionProvider)]+ (let [^ConnectionProvider provider (mdb.quartz/->ConnectionProvider)](with-open [^Connection conn (.getConnection provider)](is (instance? Connection conn))(is (not (identical? outer conn))@@ -56,7 +56,7 @@(with-open [^Connection main-conn (.getConnection ^javax.sql.DataSource app-db)];; ...and simulate being inside a `with-transaction` block on it(binding [t2.conn/*current-connectable* main-conn]- (let [^ConnectionProvider provider (task.bootstrap/->ConnectionProvider)+ (let [^ConnectionProvider provider (mdb.quartz/->ConnectionProvider)result (future(with-open [^Connection conn (.getConnection provider)](with-open [stmt (.createStatement conn)]diff --git a/test/metabase/indexed_entities/task/index_values_test.clj b/test/metabase/indexed_entities/task/index_values_test.cljindex 5650bb5e990..c05f9657318 100644--- a/test/metabase/indexed_entities/task/index_values_test.clj+++ b/test/metabase/indexed_entities/task/index_values_test.clj@@ -2,6 +2,7 @@{:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.indexed-entities.task.index-values-test]}}}}}}(:require[clojure.test :refer :all]+ [metabase.indexed-entities.models.model-index :as model-index][metabase.indexed-entities.models.model-index-test :refer [with-scheduler-setup!]][metabase.indexed-entities.task.index-values :as task.index-values][metabase.task.impl :as task]@@ -36,7 +37,7 @@(task.index-values/add-indexing-job model-index)(is (some? (get-trigger)) "Trigger should exist after creating model index");; delete the trigger from Quartz- (task.index-values/remove-indexing-job model-index)+ (model-index/remove-indexing-job model-index)(is (nil? (get-trigger)) "Trigger should be deleted");; call job-init! to recreate missing triggers(#'task.index-values/job-init!)diff --git a/test/metabase/notification/task/send_test.clj b/test/metabase/notification/task/send_test.cljindex 4228a0cd0c2..a254a6b245c 100644--- a/test/metabase/notification/task/send_test.clj+++ b/test/metabase/notification/task/send_test.clj@@ -5,6 +5,7 @@[metabase.driver :as driver][metabase.notification.models :as models.notification][metabase.notification.task.send :as task.notification]+ [metabase.notification.task.send-trigger :as notification.task.send-trigger][metabase.notification.test-util :as notification.tu][metabase.task.core :as task][metabase.test :as mt]@@ -66,13 +67,13 @@(testing "sanity check that it has triggers to begin with"(is (not-empty notification-triggers)))(testing "init send notification triggers are idempotent if the subscription doesn't change"- (task.notification/init-send-notification-triggers!)+ (notification.task.send-trigger/init-send-notification-triggers!)(is (= notification-triggers (notification.tu/send-notification-triggers subscription-id))))(testing "Re-create triggers if it's not existed"(task/delete-trigger! (TriggerKey. (-> notification-triggers first :key)))(testing "sanity check that the trigger is deleted"(is (empty? (notification.tu/send-notification-triggers subscription-id))))- (task.notification/init-send-notification-triggers!)+ (notification.task.send-trigger/init-send-notification-triggers!)(is (= notification-triggers (notification.tu/send-notification-triggers subscription-id))))(testing "deletes triggers for subscriptions that no longer exist"(let [subscription-id (first (t2/select-pks-vec :model/NotificationSubscription@@ -80,7 +81,7 @@(t2/delete! :notification_subscription subscription-id)(testing "sanity check that it has trigger before"(is (not-empty (notification.tu/send-notification-triggers subscription-id))))- (task.notification/init-send-notification-triggers!)+ (notification.task.send-trigger/init-send-notification-triggers!)(is (empty? (notification.tu/send-notification-triggers subscription-id)))))))))(deftest init-send-notification-triggers-inactive-test@@ -100,12 +101,12 @@(testing "skips triggers for inactive notifications";; Deactivate the notification(t2/update! :model/Notification (:id notification) {:active false})- (task.notification/init-send-notification-triggers!)+ (notification.task.send-trigger/init-send-notification-triggers!)(is (empty? (notification.tu/send-notification-triggers subscription-id))))(testing "recreates triggers when notification is reactivated";; Reactivate the notification(t2/update! :model/Notification (:id notification) {:active true})- (task.notification/init-send-notification-triggers!)+ (notification.task.send-trigger/init-send-notification-triggers!)(is (= notification-triggers (notification.tu/send-notification-triggers subscription-id))))))))(deftest update-send-notification-triggers-timezone-test@@ -128,7 +129,7 @@;; Change the report timezone(driver/report-timezone! new-timezone);; Call the function under test- (task.notification/update-send-notification-triggers-timezone!)+ (notification.task.send-trigger/update-send-notification-triggers-timezone!);; Verify the timezone was updated(let [updated-triggers (notification.tu/send-notification-triggers subscription-id)](is (not-empty updated-triggers))diff --git a/test/metabase/notification/test_util.clj b/test/metabase/notification/test_util.cljindex a8a8af06fd4..965ccadab5f 100644--- a/test/metabase/notification/test_util.clj+++ b/test/metabase/notification/test_util.clj@@ -15,7 +15,7 @@[metabase.notification.models :as models.notification][metabase.notification.payload.core :as notification.payload][metabase.notification.send :as notification.send]- [metabase.notification.task.send :as task.notification]+ [metabase.notification.task.send-trigger :as notification.task.send-trigger][metabase.task.core :as task][metabase.test :as mt][metabase.util :as u]@@ -261,8 +261,8 @@[subscription-id](map#(select-keys % [:key :schedule :data :timezone])- (task/existing-triggers @#'task.notification/send-notification-job-key- (#'task.notification/send-notification-trigger-key subscription-id))))+ (task/existing-triggers notification.task.send-trigger/send-notification-job-key+ (#'notification.task.send-trigger/send-notification-trigger-key subscription-id))))(defn notification-triggers"Return the quartz triggers for a notification."@@ -275,7 +275,7 @@([subscription-id cron-schedule](subscription->trigger-info subscription-id cron-schedule "UTC"))([subscription-id cron-schedule timezone]- {:key (.getName ^org.quartz.TriggerKey (#'task.notification/send-notification-trigger-key subscription-id))+ {:key (.getName ^org.quartz.TriggerKey (#'notification.task.send-trigger/send-notification-trigger-key subscription-id)):schedule cron-schedule:data {"subscription-id" subscription-id}:timezone timezone}))diff --git a/test/metabase/pulse/models/pulse_channel_test.clj b/test/metabase/pulse/models/pulse_channel_test.cljindex c01bcaadfca..40f313d2423 100644--- a/test/metabase/pulse/models/pulse_channel_test.clj+++ b/test/metabase/pulse/models/pulse_channel_test.clj@@ -4,6 +4,7 @@[medley.core :as m][metabase.pulse.models.pulse-channel :as pulse-channel][metabase.pulse.task.send-pulses :as task.send-pulses]+ [metabase.pulse.task.send-pulses-trigger :as task.send-pulses-trigger][metabase.task.core :as task][metabase.test :as mt][metabase.util :as u]@@ -356,7 +357,7 @@(defn pulse->trigger-info[pulse-id schedule-map pc-ids]- {:key (.getName ^org.quartz.TriggerKey (#'task.send-pulses/send-pulse-trigger-key pulse-id schedule-map))+ {:key (.getName ^org.quartz.TriggerKey (#'task.send-pulses-trigger/send-pulse-trigger-key pulse-id schedule-map)):schedule (u.cron/schedule-map->cron-string schedule-map):priority 6:data {"pulse-id" pulse-id@@ -376,7 +377,7 @@(defn send-pulse-triggers[pulse-id & {:keys [additional-keys]}]- (->> (task/job-info @#'task.send-pulses/send-pulse-job-key)+ (->> (task/job-info task.send-pulses-trigger/send-pulse-job-key):triggers(map #(select-keys % (concat [:key :schedule :data :priority] additional-keys)))(filter #(or (nil? pulse-id) (= pulse-id (get-in % [:data "pulse-id"]))))diff --git a/test/metabase/pulse/task/send_pulses_test.clj b/test/metabase/pulse/task/send_pulses_test.cljindex 4b7c3abbca6..5eab7f21ccb 100644--- a/test/metabase/pulse/task/send_pulses_test.clj+++ b/test/metabase/pulse/task/send_pulses_test.clj@@ -9,6 +9,7 @@[metabase.pulse.models.pulse-channel-test :as pulse-channel-test][metabase.pulse.send :as pulse.send][metabase.pulse.task.send-pulses :as task.send-pulses]+ [metabase.pulse.task.send-pulses-trigger :as task.send-pulses-trigger][metabase.task.core :as task][metabase.test :as mt][metabase.test.util :as mt.util]@@ -189,7 +190,7 @@(testing "sanity check that there are no triggers"(is (empty? (all-send-pulse-triggers))))(testing "init-send-pulse-triggers! should create triggers for each pulse-channel"- (#'task.send-pulses/init-dashboard-subscription-triggers!)+ (#'task.send-pulses-trigger/init-dashboard-subscription-triggers!)(is (=? #{(pulse-channel-test/pulse->trigger-info pulse-1 daily-at-1am [pc-1-1 pc-1-2]);; pc-2-1 has the same schedule as pc-1-1 and pc-1-2 but it's not on the same trigger because it's a;; different schedule@@ -354,7 +355,7 @@(testing "sanity check that it has triggers to begin with"(is (not-empty pulse-triggers)))(testing "init send pulse triggers are idempotent if the pulse channel doesn't change"- (#'task.send-pulses/init-dashboard-subscription-triggers!)+ (#'task.send-pulses-trigger/init-dashboard-subscription-triggers!)(is (= pulse-triggers (pulse-channel-test/send-pulse-triggers pulse-id)))))))))(deftest init-send-pulse-triggers-skip-alert-test@@ -368,4 +369,4 @@:channel_type :slack:details {:channel "#random"}}daily-at-1am)]- (is (not (contains? (set (map :id (#'task.send-pulses/active-dashsub-pcs))) pc-id)))))))+ (is (not (contains? (set (map :id (#'task.send-pulses-trigger/active-dashsub-pcs))) pc-id)))))))diff --git a/test/metabase/queries/models/card_test.clj b/test/metabase/queries/models/card_test.cljindex 9f65ceee28e..12afa0cd172 100644--- a/test/metabase/queries/models/card_test.clj+++ b/test/metabase/queries/models/card_test.clj@@ -17,6 +17,7 @@[metabase.queries.models.card :as card][metabase.queries.models.parameter-card :as parameter-card][metabase.queries.schema :as queries.schema]+ [metabase.query-processor.card :as qp.card][metabase.query-processor.card-test :as qp.card-test][metabase.query-processor.preprocess :as qp.preprocess][metabase.search.ingestion :as search.ingestion]@@ -352,7 +353,7 @@:slug "date":default nil:required false}]- (card/template-tag-parameters card))))))+ (qp.card/template-tag-parameters card))))))(deftest ^:parallel template-tag-parameters-test-2(testing "Card with a non-Field-filter parameter"@@ -364,7 +365,7 @@:slug "id":default "1":required true}]- (card/template-tag-parameters card))))))+ (qp.card/template-tag-parameters card))))))(deftest ^:parallel template-tag-parameters-test-3(testing "Should ignore native query snippets and source card IDs"@@ -376,7 +377,7 @@:slug "id":default "1":required true}]- (card/template-tag-parameters card))))))+ (qp.card/template-tag-parameters card))))))(defn- native-query-card"Build a card map with a native query containing the given template tags.@@ -403,7 +404,7 @@:slug "name":default "Alice":required false}]- (card/template-tag-parameters card))))))+ (qp.card/template-tag-parameters card))))))(deftest ^:parallel template-tag-parameters-boolean-tag-test(testing ":boolean template tag without widget-type produces :boolean/= parameter type (QUE2-326)"@@ -421,7 +422,7 @@:slug "active":default nil:required false}]- (card/template-tag-parameters card))))))+ (qp.card/template-tag-parameters card))))))(deftest ^:parallel template-tag-parameters-dimension-category-widget-test(testing ":dimension template tag with :category widget-type passes through widget-type (QUE2-326)"@@ -441,7 +442,7 @@:slug "cat":default nil:required false}]- (card/template-tag-parameters card))))))+ (qp.card/template-tag-parameters card))))))(deftest validate-template-tag-field-ids-test(testing "Disallow saving a Card with native query Field filter template tags referencing a different Database (#14145)"diff --git a/test/metabase/sso/api/ldap_test.clj b/test/metabase/sso/api/ldap_test.cljindex 74983caef80..8e4f37b2c96 100644--- a/test/metabase/sso/api/ldap_test.clj+++ b/test/metabase/sso/api/ldap_test.clj@@ -5,6 +5,7 @@[metabase.settings.core :as setting][metabase.sso.ldap :as ldap][metabase.sso.ldap-test-util :as ldap.test]+ [metabase.sso.ldap.settings :as sso.ldap.settings][metabase.sso.settings :as sso.settings][metabase.test :as mt]))@@ -52,7 +53,7 @@:ldap-port (int (ldap.test/get-ldap-port)))))(testing "Passing ldap-enabled=false will disable LDAP"(mt/user-http-request :crowberto :put 200 "ldap/settings" (ldap-test-details false))- (is (not (sso.settings/ldap-enabled))))+ (is (not (sso.ldap.settings/ldap-enabled))))(testing "Passing ldap-enabled=false still validates the LDAP settings"(mt/user-http-request :crowberto :put 500 "ldap/settings"(assoc (ldap-test-details false) :ldap-password "wrong-password")))@@ -74,8 +75,8 @@(testing "PUT /api/ldap/settings clearing the settings disables LDAP and clears the host"(ldap.test/with-ldap-server!(mt/user-http-request :crowberto :put 200 "ldap/settings" (ldap-test-details))- (is (sso.settings/ldap-enabled))+ (is (sso.ldap.settings/ldap-enabled))(mt/with-dynamic-fn-redefs [ldap/test-ldap-connection (constantly {:status :SUCCESS})](mt/user-http-request :crowberto :put 200 "ldap/settings" {:ldap-host nil :ldap-enabled false}))- (is (not (sso.settings/ldap-enabled)))+ (is (not (sso.ldap.settings/ldap-enabled)))(is (nil? (sso.settings/ldap-host))))))diff --git a/test/metabase/sso/settings_test.clj b/test/metabase/sso/settings_test.cljindex 4b66e980a38..2c3bf1d3d56 100644--- a/test/metabase/sso/settings_test.clj+++ b/test/metabase/sso/settings_test.clj@@ -3,6 +3,7 @@[clojure.test :refer :all][metabase.sso.ldap :as ldap][metabase.sso.ldap-test-util :as ldap.test]+ [metabase.sso.ldap.settings :as sso.ldap.settings][metabase.sso.settings :as sso.settings][metabase.test :as mt]))@@ -13,12 +14,12 @@(mt/with-dynamic-fn-redefs [ldap/test-current-ldap-details (constantly {:status :ERROR :message "test error"})](is (thrown-with-msg? clojure.lang.ExceptionInfo#"Unable to connect to LDAP server"- (sso.settings/ldap-enabled! true))))+ (sso.ldap.settings/ldap-enabled! true))))(mt/with-dynamic-fn-redefs [ldap/test-current-ldap-details (constantly {:status :SUCCESS})]- (sso.settings/ldap-enabled! true)- (is (sso.settings/ldap-enabled))- (sso.settings/ldap-enabled! false)- (is (not (sso.settings/ldap-enabled))))))))+ (sso.ldap.settings/ldap-enabled! true)+ (is (sso.ldap.settings/ldap-enabled))+ (sso.ldap.settings/ldap-enabled! false)+ (is (not (sso.ldap.settings/ldap-enabled))))))))(deftest ^:parallel send-new-sso-user-admin-email?-test(is ((some-fn nil? boolean?) (sso.settings/send-new-sso-user-admin-email?))diff --git a/test/metabase/sync/task/sync_databases_test.clj b/test/metabase/sync/task/sync_databases_test.cljindex 0180b326ab5..b62b65c7d0e 100644--- a/test/metabase/sync/task/sync_databases_test.clj+++ b/test/metabase/sync/task/sync_databases_test.clj@@ -10,6 +10,7 @@[metabase.sync.field-values :as sync.field-values][metabase.sync.schedules :as sync.schedules][metabase.sync.task.sync-databases :as task.sync-databases]+ [metabase.sync.task.sync-databases-trigger :as sync-databases-trigger][metabase.task.core :as task][metabase.test :as mt][metabase.test.util :as tu]@@ -44,8 +45,8 @@"Returns the name of trigger for DB.These are all the trigger names that a database SHOULD have."[db]- (set (map #(.getName ^TriggerKey (#'task.sync-databases/trigger-key (t2/instance :model/Database db) %))- @#'task.sync-databases/all-tasks)))+ (set (map #(.getName ^TriggerKey (#'sync-databases-trigger/trigger-key (t2/instance :model/Database db) %))+ @#'sync-databases-trigger/all-tasks)))(defn query-all-db-sync-triggers-name"Find the all triggers for DB \"db\".@@ -53,10 +54,10 @@[db](let [db (t2/instance :model/Database db)](assert (some? (#'task/scheduler)) "makes sure the scheduler is initialized!")- (->> (for [task-info @#'task.sync-databases/all-tasks]- (keep #(when (= (.getName ^TriggerKey (#'task.sync-databases/trigger-key db task-info)) (:key %))+ (->> (for [task-info @#'sync-databases-trigger/all-tasks]+ (keep #(when (= (.getName ^TriggerKey (#'sync-databases-trigger/trigger-key db task-info)) (:key %))(:key %))- (:triggers (task/job-info (#'task.sync-databases/job-key task-info)))))+ (:triggers (task/job-info (#'sync-databases-trigger/job-key task-info)))))flattenset)))@@ -221,14 +222,14 @@(mt/with-temp [:model/Database non-stub {:is_stub false}:model/Database stub {:is_stub true}](let [calls (atom 0)]- (with-redefs [task.sync-databases/update-db-trigger-if-needed! (fn [_ _] (swap! calls inc))]+ (with-redefs [sync-databases-trigger/update-db-trigger-if-needed! (fn [_ _] (swap! calls inc))](testing "non-stub: triggers are considered for scheduling"(reset! calls 0)- (task.sync-databases/check-and-schedule-tasks-for-db! non-stub)+ (sync-databases-trigger/check-and-schedule-tasks-for-db! non-stub)(is (pos? @calls)))(testing "stub: no scheduling calls are made"(reset! calls 0)- (task.sync-databases/check-and-schedule-tasks-for-db! stub)+ (sync-databases-trigger/check-and-schedule-tasks-for-db! stub)(is (zero? @calls))))))))(deftest check-orphaned-jobs-removed-testdiff --git a/test/metabase/warehouses_rest/api_test.clj b/test/metabase/warehouses_rest/api_test.cljindex 8c2ab511c19..856601110d4 100644--- a/test/metabase/warehouses_rest/api_test.clj+++ b/test/metabase/warehouses_rest/api_test.clj@@ -30,6 +30,7 @@[metabase.sync.sync-metadata :as sync-metadata][metabase.sync.task.sync-databases :as task.sync-databases][metabase.sync.task.sync-databases-test :as task.sync-databases-test]+ [metabase.sync.task.sync-databases-trigger :as sync-databases-trigger][metabase.task.core :as task][metabase.test :as mt][metabase.test.data.impl :as data.impl]@@ -373,7 +374,7 @@(defn- sync-and-analyze-trigger-name[db]- (.getName ^TriggerKey (#'task.sync-databases/trigger-key db @#'task.sync-databases/sync-analyze-task-info)))+ (.getName ^TriggerKey (#'sync-databases-trigger/trigger-key db sync-databases-trigger/sync-analyze-task-info)))(defmacro with-test-driver-available![& body]