Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Break requiring resolve cycles (#82011)

Ben Grabow30346d6a64a4
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 43dbd0f726a..84472144430 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -887,7 +887,7 @@
metabase.driver.settings
metabase.driver.sql
metabase.driver.sql-jdbc
- metabase.driver.sql-jdbc.connection
+ metabase.driver.sql-jdbc.connection.pool-lock
metabase.driver.sql-jdbc.execute
metabase.driver.sql-jdbc.execute.diagnostic
metabase.driver.sql-jdbc.sync
@@ -1817,6 +1817,7 @@
:api #{metabase.permissions.core
metabase.permissions.init
metabase.permissions.metric
+ metabase.permissions.path
metabase.permissions.schema}
:uses #{api
app-db
@@ -1881,6 +1882,7 @@
{:team "UX West"
:api #{metabase.premium-features.api
metabase.premium-features.core
+ metabase.premium-features.defenterprise
metabase.premium-features.init}
:uses #{analytics.interface
analytics.internal-stats
@@ -2625,7 +2627,7 @@
metabase.sync.field-values
metabase.sync.init
metabase.sync.schedules
- metabase.sync.task.sync-databases
+ metabase.sync.task.sync-databases-trigger
metabase.sync.util}
:uses #{analytics.interface
api
diff --git a/enterprise/backend/src/metabase_enterprise/advanced_config/file/databases.clj b/enterprise/backend/src/metabase_enterprise/advanced_config/file/databases.clj
index 3182e16e441..c06d20007e2 100644
--- a/enterprise/backend/src/metabase_enterprise/advanced_config/file/databases.clj
+++ b/enterprise/backend/src/metabase_enterprise/advanced_config/file/databases.clj
@@ -8,6 +8,7 @@
[metabase-enterprise.advanced-config.settings :as advanced-config.settings]
[metabase.driver.util :as driver.u]
[metabase.sample-data.core :as sample-data]
+ [metabase.sync.core :as sync]
[metabase.util :as u]
[metabase.util.log :as log]
[metabase.util.quick-task :as quick-task]))
@@ -137,8 +138,7 @@
(log/info "Created stub database; skipping sync.")
(advanced-config.settings/config-from-file-sync-databases)
- (let [sync-database! (requiring-resolve 'metabase.sync.core/sync-database!)]
- (quick-task/submit-task! (fn [] (sync-database! db))))
+ (quick-task/submit-task! (fn [] (sync/sync-database! db)))
:else
(log/info "Sync on database creation when initializing from file is disabled. Skipping sync."))))))))
diff --git a/enterprise/backend/src/metabase_enterprise/sandbox/models/sandbox.clj b/enterprise/backend/src/metabase_enterprise/sandbox/models/sandbox.clj
index e0a813659be..fecdd733cf7 100644
--- a/enterprise/backend/src/metabase_enterprise/sandbox/models/sandbox.clj
+++ b/enterprise/backend/src/metabase_enterprise/sandbox/models/sandbox.clj
@@ -20,6 +20,7 @@
[metabase.permissions.core :as perms]
[metabase.premium-features.core :refer [defenterprise]]
[metabase.query-processor.error-type :as qp.error-type]
+ [metabase.query-processor.preprocess :as qp.preprocess]
[metabase.query-processor.schema :as qp.schema]
[metabase.request.core :as request]
[metabase.util :as u]
@@ -46,7 +47,7 @@
"Return a mapping of field names to corresponding cols for given table."
[table-id]
(into {} (for [col (request/as-admin
- ((requiring-resolve 'metabase.query-processor.preprocess/query->expected-cols)
+ (qp.preprocess/query->expected-cols
{:database (database/table-id->database-id table-id)
:type :query
:query {:source-table table-id}}))]
diff --git a/enterprise/backend/src/metabase_enterprise/sandbox/query_processor/middleware/sandboxing.clj b/enterprise/backend/src/metabase_enterprise/sandbox/query_processor/middleware/sandboxing.clj
index 948788052ff..493ded4fc51 100644
--- a/enterprise/backend/src/metabase_enterprise/sandbox/query_processor/middleware/sandboxing.clj
+++ b/enterprise/backend/src/metabase_enterprise/sandbox/query_processor/middleware/sandboxing.clj
@@ -24,8 +24,10 @@
[metabase.lib.schema.util :as lib.schema.util]
[metabase.lib.walk :as lib.walk]
[metabase.premium-features.core :as premium-features :refer [defenterprise]]
+ [metabase.query-processor :as qp]
[metabase.query-processor.error-type :as qp.error-type]
[metabase.query-processor.pipeline :as qp.pipeline]
+ [metabase.query-processor.preprocess :as qp.preprocess]
[metabase.query-processor.util.persisted-cache :as qp.persisted]
[metabase.request.core :as request]
[metabase.util :as u]
@@ -148,15 +150,14 @@
(try
(lib/without-cleaning
(fn []
- (let [preprocess (requiring-resolve 'metabase.query-processor.preprocess/preprocess)]
- (request/as-admin
- ;; preprocessing normally loses metadata attached to the last stage, since legacy MBQL syntax does not
- ;; support it and preprocessing roundtrips to legacy and back a few times... to make sure it's preserved,
- ;; append an extra dummy stage before preprocessing and then toss it when we're done.
- (-> query
- lib/append-stage
- preprocess
- (update :stages pop))))))
+ (request/as-admin
+ ;; preprocessing normally loses metadata attached to the last stage, since legacy MBQL syntax does not
+ ;; support it and preprocessing roundtrips to legacy and back a few times... to make sure it's preserved,
+ ;; append an extra dummy stage before preprocessing and then toss it when we're done.
+ (-> query
+ lib/append-stage
+ qp.preprocess/preprocess
+ (update :stages pop)))))
(catch Throwable e
(throw (ex-info (tru "Error preprocessing query when applying Sandbox: {0}" (ex-message e))
{:query query}
@@ -204,8 +205,7 @@
;; to a custom handler, and we don't want to accidentally terminate the stream here!
(binding [qp.pipeline/*result* qp.pipeline/default-result-handler]
(request/as-admin
- ((requiring-resolve 'metabase.query-processor/process-query)
- query)))]
+ (qp/process-query query)))]
(when-not (= (:status result) :completed)
(throw (ex-info "Error running query to determine metadata"
{:query query, :result result})))
diff --git a/frontend/src/metabase-lib/v1/parameters/utils/template-tags.ts b/frontend/src/metabase-lib/v1/parameters/utils/template-tags.ts
index 86dc3f9ac02..d510acbdd53 100644
--- a/frontend/src/metabase-lib/v1/parameters/utils/template-tags.ts
+++ b/frontend/src/metabase-lib/v1/parameters/utils/template-tags.ts
@@ -65,7 +65,7 @@ export function getTemplateTagParameter(
};
}
-// NOTE: this should mirror `template-tag-parameters` in src/metabase/queries/models/card.clj
+// NOTE: this should mirror `template-tag-parameters` in src/metabase/query_processor/card.clj
// If this function moves you should update the comment that links to this one
export function getTemplateTagParameters(
tags: TemplateTag[],
diff --git a/src/metabase/actions/schema.clj b/src/metabase/actions/schema.clj
index 5d9506c01d1..a2be7f1bb31 100644
--- a/src/metabase/actions/schema.clj
+++ b/src/metabase/actions/schema.clj
@@ -1,5 +1,6 @@
(ns metabase.actions.schema
(:require
+ [metabase.actions.http-action :as http-action]
[metabase.lib-be.schema :as lib-be.schema]
[metabase.lib.core :as lib]
[metabase.lib.schema.common :as lib.schema.common]
@@ -94,7 +95,7 @@
[:fn
{:error/fn (fn [_ _]
(deferred-tru "must be a valid json-query, something like ''.item.title''"))}
- #((requiring-resolve 'metabase.actions.http-action/apply-json-query) {} %)]])
+ #(http-action/apply-json-query {} %)]])
(mr/def ::http-action.template
[:map {:closed true}
diff --git a/src/metabase/analytics/prometheus.clj b/src/metabase/analytics/prometheus.clj
index 26e1df4916e..eab04887eb6 100644
--- a/src/metabase/analytics/prometheus.clj
+++ b/src/metabase/analytics/prometheus.clj
@@ -15,6 +15,8 @@
[jvm-alloc-rate-meter.core :as alloc-rate-meter]
[jvm-hiccup-meter.core :as hiccup-meter]
[metabase.analytics-interface.core :as analytics.interface]
+ ;; We should not be using specific driver implementations
+ [metabase.driver.sql-jdbc.connection.pool-lock :as pool-lock]
[metabase.util :as u]
[metabase.util.i18n :refer [trs]]
[metabase.util.log :as log]
@@ -135,20 +137,18 @@
arr))
(defn- conn-pool-bean-diag-info [acc ^ObjectName jmx-bean]
- ;; We should not be using specific driver implementations
- (let [pool-var (requiring-resolve 'metabase.driver.sql-jdbc.connection/pool-cache-key->connection-pool)]
- ;; Using this `locking` is non-obvious but absolutely required to avoid the deadlock inside c3p0 implementation. The
- ;; act of JMX attribute reading first locks a DynamicPooledDataSourceManagerMBean object, and then a
- ;; PoolBackedDataSource object. Conversely, the act of creating a pool (with
- ;; com.mchange.v2.c3p0.DataSources/pooledDataSource) first locks PoolBackedDataSource and then
- ;; DynamicPooledDataSourceManagerMBean. We have to lock a common monitor (which `pool-cache-key->connection-pool` is)
- ;; to prevent the deadlock. Hopefully.
- ;; Issue against c3p0: https://github.com/swaldman/c3p0/issues/95
- (locking @pool-var
- (let [bean-id (.getCanonicalName jmx-bean)
- props [:numConnections :numIdleConnections :numBusyConnections
- :minPoolSize :maxPoolSize :numThreadsAwaitingCheckoutDefaultUser]]
- (assoc acc (jmx/read bean-id :dataSourceName) (jmx/read bean-id props))))))
+ ;; Using this `locking` is non-obvious but absolutely required to avoid the deadlock inside c3p0 implementation. The
+ ;; act of JMX attribute reading first locks a DynamicPooledDataSourceManagerMBean object, and then a
+ ;; PoolBackedDataSource object. Conversely, the act of creating a pool (with
+ ;; com.mchange.v2.c3p0.DataSources/pooledDataSource) first locks PoolBackedDataSource and then
+ ;; DynamicPooledDataSourceManagerMBean. We have to lock a common monitor (which `pool-lock/monitor` is)
+ ;; to prevent the deadlock. Hopefully.
+ ;; Issue against c3p0: https://github.com/swaldman/c3p0/issues/95
+ (locking pool-lock/monitor
+ (let [bean-id (.getCanonicalName jmx-bean)
+ props [:numConnections :numIdleConnections :numBusyConnections
+ :minPoolSize :maxPoolSize :numThreadsAwaitingCheckoutDefaultUser]]
+ (assoc acc (jmx/read bean-id :dataSourceName) (jmx/read bean-id props)))))
(defn connection-pool-info
"Builds a map of info about the current c3p0 connection pools managed by this Metabase instance."
diff --git a/src/metabase/api/docs.clj b/src/metabase/api/docs.clj
index 010f891977c..ad77928d72c 100644
--- a/src/metabase/api/docs.clj
+++ b/src/metabase/api/docs.clj
@@ -3,83 +3,25 @@
(:require
[clojure.java.io :as io]
[clojure.string :as str]
- [clojure.walk :as walk]
[compojure.core :refer [GET]]
+ [metabase.api.docs.regenerate]
[metabase.api.open-api :as open-api]
[metabase.api.util.handlers :as handlers]
[metabase.config.core :as config]
[metabase.util.json :as json]
[metabase.util.log :as log]
+ [potemkin :as p]
[ring.middleware.content-type :as content-type]
[ring.util.response :as response]))
(set! *warn-on-reflection* true)
-(def openapi-file-path
- "Path to the local OpenAPI specification file."
- "resources/openapi/openapi.json")
-
-(defn- sort-keys
- "Sort maps and sets alphabetically to reduce diff noise on openapi.json"
- [data]
- (walk/postwalk
- (fn [x]
- (cond
- (map? x) (into (sorted-map) x)
- (set? x) (into (sorted-set) x)
- :else x))
- data))
-
-(defn write-openapi-spec-to-file!
- "Generate and write the OpenAPI specification to a local file.
- Takes the root handler and generates the complete OpenAPI spec, writing it to [[openapi-file-path]]."
- [root-handler]
- (try
- (let [spec (merge
- (open-api/root-open-api-object root-handler)
- {:servers [{:url ""
- :description "Metabase API"}]})
- file (io/file openapi-file-path)]
- ;; Create parent directory if it doesn't exist
- (when-let [parent-dir (.getParentFile file)]
- (.mkdirs parent-dir))
- (json/encode-to (sort-keys spec) (io/writer file) {:pretty true})
- (log/info "OpenAPI specification written to" openapi-file-path))
- (catch Throwable e
- (log/errorf "Failed to write OpenAPI specification to file: %s" (ex-message e)))))
-
-(defonce ^:private openapi-regen-state
- (atom {:executing? false
- :needs-regen? false}))
-
-(def ^:private debounce-delay-ms
- "Delay in milliseconds before starting regeneration to allow multiple rapid requests to coalesce."
- 200)
-
-(defn request-spec-regeneration!
- "Request OpenAPI spec regeneration. Multiple rapid requests are coalesced into one execution.
- Uses a debounce delay to batch requests that arrive in quick succession (e.g., when re-evaling a file with multiple endpoints)."
- [routes]
- (swap! openapi-regen-state assoc :needs-regen? true)
-
- (when-not (:executing? @openapi-regen-state)
- (swap! openapi-regen-state assoc :executing? true)
- (future
- (try
- ;; Wait for debounce period to let multiple requests coalesce
- (Thread/sleep ^Long debounce-delay-ms)
- (loop []
- (when (:needs-regen? @openapi-regen-state)
- (swap! openapi-regen-state assoc :needs-regen? false)
- (log/info "Regenerating OpenAPI specification...")
- (write-openapi-spec-to-file! routes)
- (log/info "OpenAPI specification regenerated successfully")
- ;; Check if more requests came in while we were working
- (recur)))
- (catch Throwable e
- (log/errorf "Error regenerating OpenAPI specification: %s" (ex-message e)))
- (finally
- (swap! openapi-regen-state assoc :executing? false))))))
+;; kept on this namespace so existing callers (metabase.api-routes.cmd) are unaffected by the split
+(p/import-vars
+ [metabase.api.docs.regenerate
+ openapi-file-path
+ request-spec-regeneration!
+ write-openapi-spec-to-file!])
(defn- read-openapi-spec-from-file
"Read the OpenAPI specification from the local file.
diff --git a/src/metabase/api/docs/regenerate.clj b/src/metabase/api/docs/regenerate.clj
new file mode 100644
index 00000000000..1e308c9e0d4
--- /dev/null
+++ b/src/metabase/api/docs/regenerate.clj
@@ -0,0 +1,80 @@
+(ns metabase.api.docs.regenerate
+ "Writing `openapi.json` to disk, and the debounced scheduler that requests a rewrite.
+
+ Split out of [[metabase.api.docs]] so that [[metabase.api.macros]] can name
+ [[request-spec-regeneration!]] directly: `metabase.api.docs` reaches `metabase.api.macros` through
+ [[metabase.api.util.handlers]], and nothing here does."
+ (:require
+ [clojure.java.io :as io]
+ [clojure.walk :as walk]
+ [metabase.api.open-api :as open-api]
+ [metabase.util.json :as json]
+ [metabase.util.log :as log]))
+
+(set! *warn-on-reflection* true)
+
+(def openapi-file-path
+ "Path to the local OpenAPI specification file."
+ "resources/openapi/openapi.json")
+
+(defn- sort-keys
+ "Sort maps and sets alphabetically to reduce diff noise on openapi.json"
+ [data]
+ (walk/postwalk
+ (fn [x]
+ (cond
+ (map? x) (into (sorted-map) x)
+ (set? x) (into (sorted-set) x)
+ :else x))
+ data))
+
+(defn write-openapi-spec-to-file!
+ "Generate and write the OpenAPI specification to a local file.
+ Takes the root handler and generates the complete OpenAPI spec, writing it to [[openapi-file-path]]."
+ [root-handler]
+ (try
+ (let [spec (merge
+ (open-api/root-open-api-object root-handler)
+ {:servers [{:url ""
+ :description "Metabase API"}]})
+ file (io/file openapi-file-path)]
+ ;; Create parent directory if it doesn't exist
+ (when-let [parent-dir (.getParentFile file)]
+ (.mkdirs parent-dir))
+ (json/encode-to (sort-keys spec) (io/writer file) {:pretty true})
+ (log/info "OpenAPI specification written to" openapi-file-path))
+ (catch Throwable e
+ (log/errorf "Failed to write OpenAPI specification to file: %s" (ex-message e)))))
+
+(defonce ^:private openapi-regen-state
+ (atom {:executing? false
+ :needs-regen? false}))
+
+(def ^:private debounce-delay-ms
+ "Delay in milliseconds before starting regeneration to allow multiple rapid requests to coalesce."
+ 200)
+
+(defn request-spec-regeneration!
+ "Request OpenAPI spec regeneration. Multiple rapid requests are coalesced into one execution.
+ Uses a debounce delay to batch requests that arrive in quick succession (e.g., when re-evaling a file with multiple endpoints)."
+ [routes]
+ (swap! openapi-regen-state assoc :needs-regen? true)
+
+ (when-not (:executing? @openapi-regen-state)
+ (swap! openapi-regen-state assoc :executing? true)
+ (future
+ (try
+ ;; Wait for debounce period to let multiple requests coalesce
+ (Thread/sleep ^Long debounce-delay-ms)
+ (loop []
+ (when (:needs-regen? @openapi-regen-state)
+ (swap! openapi-regen-state assoc :needs-regen? false)
+ (log/info "Regenerating OpenAPI specification...")
+ (write-openapi-spec-to-file! routes)
+ (log/info "OpenAPI specification regenerated successfully")
+ ;; Check if more requests came in while we were working
+ (recur)))
+ (catch Throwable e
+ (log/errorf "Error regenerating OpenAPI specification: %s" (ex-message e)))
+ (finally
+ (swap! openapi-regen-state assoc :executing? false))))))
diff --git a/src/metabase/api/macros.clj b/src/metabase/api/macros.clj
index 3071ce55d26..ea041a2a8ca 100644
--- a/src/metabase/api/macros.clj
+++ b/src/metabase/api/macros.clj
@@ -27,6 +27,7 @@
[malli.util]
[medley.core :as m]
[metabase.api.common.internal]
+ [metabase.api.docs.regenerate :as docs.regenerate]
[metabase.api.macros.defendpoint.closed-schemas :as closed-schemas]
[metabase.api.macros.defendpoint.open-api]
[metabase.api.macros.scope]
@@ -819,8 +820,7 @@
(when (config/config-bool :mb-enable-openapi-auto-regen)
(try
(events/publish-event! :event/api-handler-update
- {:api.docs/request-rebuild
- (requiring-resolve 'metabase.api.docs/request-spec-regeneration!)})
+ {:api.docs/request-rebuild #'docs.regenerate/request-spec-regeneration!})
(catch Throwable e
(log/debugf "Failed to publish api-handler-update event: %s" (ex-message e)))))))
diff --git a/src/metabase/app_db/custom_migrations.clj b/src/metabase/app_db/custom_migrations.clj
index 224b44e84cd..0312dcc0687 100644
--- a/src/metabase/app_db/custom_migrations.clj
+++ b/src/metabase/app_db/custom_migrations.clj
@@ -28,9 +28,9 @@
[metabase.app-db.custom-migrations.pulse-to-notification :as pulse-to-notification]
[metabase.app-db.custom-migrations.reserve-at-symbol-user-attributes :as reserve-at-symbol-user-attributes]
[metabase.app-db.custom-migrations.util :as custom-migrations.util]
+ [metabase.app-db.quartz]
[metabase.app-db.setting :as mdb.setting]
[metabase.config.core :as config]
- [metabase.task.bootstrap]
[metabase.util.date-2 :as u.date]
[metabase.util.encryption :as encryption]
[metabase.util.honey-sql-2 :as h2x]
@@ -1760,7 +1760,7 @@
;; on migrate up:
;; - migrate alerts from pulse table to notification table
;; - And then on startup new send notification triggers are created by running
-;; [[metabase.notification.task.send/init-send-notification-triggers!]]
+;; [[metabase.notification.task.send-trigger/init-send-notification-triggers!]]
(define-migration MigrateAlertToNotification
(pulse-to-notification/migrate-alerts!))
diff --git a/src/metabase/app_db/data_source.clj b/src/metabase/app_db/data_source.clj
index 227577ed2b7..35f41ce865a 100644
--- a/src/metabase/app_db/data_source.clj
+++ b/src/metabase/app_db/data_source.clj
@@ -6,6 +6,7 @@
[clojure.string :as str]
[metabase.app-db.spec :as mdb.spec]
[metabase.app-db.update-h2 :as update-h2]
+ [metabase.auth-provider.core :as auth-provider]
[metabase.config.core :as config]
[metabase.connection-pool :as connection-pool]
[metabase.util.log :as log]
@@ -25,10 +26,10 @@
(defn- renew-azure-managed-identity-password
[client-id]
(let [{:keys [access_token expires_in]}
- ((requiring-resolve 'metabase.auth-provider.core/fetch-auth) :azure-managed-identity nil {:azure-managed-identity-client-id client-id})]
+ (auth-provider/fetch-auth :azure-managed-identity nil {:azure-managed-identity-client-id client-id})]
{:password access_token
:expiry (+ (*current-millis*) (* (- (parse-long expires_in)
- @(requiring-resolve 'metabase.auth-provider.core/azure-auth-token-renew-slack-seconds))
+ auth-provider/azure-auth-token-renew-slack-seconds)
1000))}))
(defn- ensure-azure-managed-identity-password
diff --git a/src/metabase/app_db/quartz.clj b/src/metabase/app_db/quartz.clj
new file mode 100644
index 00000000000..4fc65070396
--- /dev/null
+++ b/src/metabase/app_db/quartz.clj
@@ -0,0 +1,29 @@
+(ns metabase.app-db.quartz
+ "Quartz's `ConnectionProvider` for the application DB."
+ (:require
+ [metabase.app-db.connection :as mdb.connection]
+ [metabase.task.bootstrap :as task.bootstrap]))
+
+(set! *warn-on-reflection* true)
+
+;; Custom `ConnectionProvider` implementation that uses a dedicated connection pool for the application DB to provide
+;; connections.
+(defrecord ^:private ConnectionProvider []
+ org.quartz.utils.ConnectionProvider
+ (initialize [_])
+ (getConnection [_]
+ ;; get a connection from the dedicated Quartz connection pool. Quartz will close it (i.e., return it to the pool)
+ ;; when it's done.
+ ;;
+ ;; very important! Fetch a new connection from the connection pool rather than reusing a Connection already bound
+ ;; to the calling thread (e.g. toucan2's *current-connectable*) -- Quartz manages the connection's whole
+ ;; lifecycle (setAutoCommit/commit/rollback/close), and its cluster locking relies on commit/rollback to release
+ ;; row locks on the QRTZ_LOCKS table, so it must never share a connection with an outer transaction.
+ ;;
+ ;; the pool is separate from the main application DB pool so that a Quartz operation triggered by a thread inside
+ ;; a `with-transaction` block can't deadlock when application code has saturated the main pool.
+ (task.bootstrap/intercept-connection (.getConnection (mdb.connection/quartz-data-source))))
+ (shutdown [_]))
+
+(when-not *compile-files*
+ (System/setProperty "org.quartz.dataSource.db.connectionProvider.class" (.getName ConnectionProvider)))
diff --git a/src/metabase/auth_provider/impl.clj b/src/metabase/auth_provider/impl.clj
index d5c94f1726b..89ce6f81227 100644
--- a/src/metabase/auth_provider/impl.clj
+++ b/src/metabase/auth_provider/impl.clj
@@ -1,6 +1,6 @@
(ns metabase.auth-provider.impl
(:require
- [metabase.premium-features.core :refer [defenterprise]]))
+ [metabase.premium-features.defenterprise :refer [defenterprise]]))
(def azure-auth-token-renew-slack-seconds
"How many seconds before expiry we should prefer renewal.
diff --git a/src/metabase/channel/email/result_attachment.clj b/src/metabase/channel/email/result_attachment.clj
index de7a7ec05bd..24f2f87856e 100644
--- a/src/metabase/channel/email/result_attachment.clj
+++ b/src/metabase/channel/email/result_attachment.clj
@@ -2,6 +2,7 @@
(:require
[clojure.java.io :as io]
[metabase.channel.render.util :as render.util]
+ [metabase.channel.shared :as channel.shared]
[metabase.driver :as driver]
[metabase.driver.util :as driver.u]
[metabase.lib.schema.id :as lib.schema.id]
@@ -101,8 +102,7 @@
(when (and (or (:include_csv card) (:include_xls card))
(pos-int? (:row_count result))
(not= (perms/download-perms-level (:dataset_query card) creator-id) :no))
- (let [maybe-realize-data-rows (requiring-resolve 'metabase.channel.shared/maybe-realize-data-rows)
- result (:result (maybe-realize-data-rows part))
+ (let [result (:result (channel.shared/maybe-realize-data-rows part))
;; the dashboard-level title override (incl. the visualizer's) names the attachment, matching the card's
;; displayed title; falls back to the card's own name (see [[render.util/dashcard-title]]).
filename-prefix (render.util/dashcard-title card dashcard)]
diff --git a/src/metabase/collections/models/collection/root.clj b/src/metabase/collections/models/collection/root.clj
index 43d373181ce..510c86c3414 100644
--- a/src/metabase/collections/models/collection/root.clj
+++ b/src/metabase/collections/models/collection/root.clj
@@ -2,7 +2,7 @@
(:require
[medley.core :as m]
[metabase.models.interface :as mi]
- [metabase.permissions.core :as perms]
+ [metabase.permissions.path :as permissions.path]
[metabase.premium-features.core :as premium-features]
[metabase.util :as u]
[metabase.util.i18n :refer [tru]]
@@ -37,8 +37,8 @@
(not (premium-features/enable-snippet-collections?)))
#{}
#{((case read-or-write
- :read perms/collection-read-path
- :write perms/collection-readwrite-path) collection)}))
+ :read permissions.path/collection-read-path
+ :write permissions.path/collection-readwrite-path) collection)}))
(def ^RootCollection root-collection
"Special placeholder object representing the Root Collection, which isn't really a real Collection."
diff --git a/src/metabase/dashboards/models/dashboard.clj b/src/metabase/dashboards/models/dashboard.clj
index 3a1b5dbece5..08d568cb202 100644
--- a/src/metabase/dashboards/models/dashboard.clj
+++ b/src/metabase/dashboards/models/dashboard.clj
@@ -27,6 +27,7 @@
[metabase.search.core :as search]
[metabase.settings.core :as setting]
[metabase.staleness.core :as staleness]
+ [metabase.sync.field-values :as sync.field-values]
[metabase.util :as u]
[metabase.util.embed :refer [maybe-populate-initially-published-at]]
[metabase.util.honey-sql-2 :as h2x]
@@ -225,7 +226,7 @@
(log/info "Referenced Fields in Dashboard params have changed: Was:" old-param-field-ids
"Is Now:" new-param-field-ids
"Newly Added:" newly-added-param-field-ids)
- ((requiring-resolve 'metabase.sync.field-values/update-field-values-for-on-demand-dbs!) newly-added-param-field-ids))))
+ (sync.field-values/update-field-values-for-on-demand-dbs! newly-added-param-field-ids))))
(defn add-dashcards!
"Add Cards to a Dashboard.
@@ -368,31 +369,9 @@
(cond-> dashboard
(collections/remote-synced-collection? parent-collection-id) collections/check-non-remote-synced-dependencies))))
-(def ^:private ParamWithMapping
- [:map
- [:id ms/NonBlankString]
- [:name ms/NonBlankString]
- [:mappings [:maybe [:set ::parameters.schema/parameter-mapping-with-dashcard]]]])
-
-(mu/defn dashboard->resolved-params :- [:map-of ms/NonBlankString ParamWithMapping]
- "Return map of Dashboard parameter key -> param with resolved `:mappings` (see the `:resolved-params` hydration
- below for an example). Callers that only need the mappings (e.g. the QP) can pass slim dashcards instead of paying
- for the full hydration."
- [dashboard :- [:map
- [:parameters [:maybe [:sequential :map]]]
- [:dashcards [:maybe [:sequential [:map
- [:parameter_mappings [:maybe [:sequential :map]]]]]]]]]
- (let [param-key->mappings (apply
- merge-with set/union
- (for [dashcard (:dashcards dashboard)
- param (:parameter_mappings dashcard)]
- {(:parameter_id param) #{(assoc param :dashcard dashcard)}}))]
- (into {} (for [{param-key :id, :as param} (:parameters dashboard)]
- [(u/qualified-name param-key) (assoc param :mappings (get param-key->mappings param-key))]))))
-
(methodical/defmethod t2/batched-hydrate [:model/Dashboard :resolved-params]
"Return map of Dashboard parameter key -> param with resolved `:mappings`.
- (dashboard->resolved-params (t2/select-one Dashboard :id 62))
+ (params/dashboard->resolved-params (t2/select-one Dashboard :id 62))
;; ->
{\"ee876336\" {:name \"Category Name\"
:slug \"category_name\"
@@ -412,7 +391,7 @@
:target [:dimension [:field-id 264]]}}}}"
[_model k dashboards]
(let [dashboards-with-cards (t2/hydrate dashboards [:dashcards :card :series])]
- (map #(assoc %1 k %2) dashboards (map dashboard->resolved-params dashboards-with-cards))))
+ (map #(assoc %1 k %2) dashboards (map params/dashboard->resolved-params dashboards-with-cards))))
(defmethod mi/exclude-internal-content-hsql :model/Dashboard
[_model & {:keys [table-alias]}]
diff --git a/src/metabase/driver/init.clj b/src/metabase/driver/init.clj
index 385645cacf1..993852c6f14 100644
--- a/src/metabase/driver/init.clj
+++ b/src/metabase/driver/init.clj
@@ -15,4 +15,5 @@
[metabase.driver.settings]
;; for the `:sql-jdbc` implementation of [[metabase.driver/incorporate-ssh-tunnel-details]]
[metabase.driver.sql-jdbc.connection.ssh-tunnel]
- [metabase.driver.sqlite]))
+ [metabase.driver.sqlite]
+ [metabase.driver.util.settings]))
diff --git a/src/metabase/driver/settings.clj b/src/metabase/driver/settings.clj
index d7141d3ae72..c727c87aed3 100644
--- a/src/metabase/driver/settings.clj
+++ b/src/metabase/driver/settings.clj
@@ -261,15 +261,6 @@
:type :integer
:default 50000)
-(defsetting engines
- "Available database engines"
- :encryption :no
- :visibility :public
- :setter :none
- :getter (fn []
- ((requiring-resolve 'metabase.driver.util/available-drivers-info)))
- :doc false)
-
(defsetting sync-leaf-fields-limit
(deferred-tru
(str "Maximum number of leaf fields synced per collection of document database. Currently relevant for Mongo."
diff --git a/src/metabase/driver/sql/query_processor.clj b/src/metabase/driver/sql/query_processor.clj
index 3a949fb82e3..fb9cbafb795 100644
--- a/src/metabase/driver/sql/query_processor.clj
+++ b/src/metabase/driver/sql/query_processor.clj
@@ -12,6 +12,7 @@
[metabase.driver-api.core :as driver-api]
[metabase.driver.common :as driver.common]
[metabase.driver.sql.query-processor.deprecated :as sql.qp.deprecated]
+ [metabase.driver.sql.query-processor.format :as sql.qp.format]
[metabase.lib.core :as lib]
[metabase.lib.metadata.protocols :as lib.metadata.protocols]
[metabase.lib.options :as lib.options]
@@ -26,6 +27,7 @@
[metabase.util.malli :as mu]
[metabase.util.match :as match]
[metabase.util.performance :as perf :refer [empty? every? get-in mapv not-empty select-keys some]]
+ [potemkin :as p]
[toucan2.pipeline :as t2.pipeline])
(:import
(java.time LocalDate LocalDateTime LocalTime OffsetDateTime OffsetTime ZonedDateTime)
@@ -33,6 +35,11 @@
(set! *warn-on-reflection* true)
+(p/import-vars
+ [sql.qp.format
+ format-honeysql
+ quote-style])
+
(def source-query-alias
"Alias to use for source queries, e.g.:
@@ -528,22 +535,6 @@
(h2x/with-database-type-info (or (h2x/database-type day-of-week-honeysql-expr)
"integer")))))))
-(defmulti quote-style
- "Return the dialect that should be used by Honey SQL 2 when building a SQL statement. Defaults to `:ansi`, but other
- valid options are `:mysql`, `:sqlserver`, `:oracle`, and `:h2` (added in
- [[metabase.util.honey-sql-2]]; like `:ansi`, but uppercases the result). Check [[honey.sql/dialects]] for all
- available dialects, or register a custom one with [[honey.sql/register-dialect!]].
-
- (honey.sql/format ... :quoting (quote-style driver), :allow-dashed-names? true)
-
- (The name of this method reflects Honey SQL 1 terminology, where \"dialect\" was called \"quote style\". To avoid
- needless churn, I haven't changed it yet. -- Cam)"
- {:added "0.32.0" :arglists '([driver])}
- driver/dispatch-on-initialized-driver
- :hierarchy #'driver/hierarchy)
-
-(defmethod quote-style :sql [_] :ansi)
-
(defmulti unix-timestamp->honeysql
"Return a HoneySQL form appropriate for converting a Unix timestamp integer field or value to an proper SQL Timestamp.
`seconds-or-milliseconds` refers to the resolution of the int in question and with be either `:seconds` or
@@ -2065,39 +2056,6 @@
(sort-by (fn [clause] [(get top-level-clause-application-order clause Integer/MAX_VALUE) clause])
(keys inner-query)))
-(defn- format-honeysql-2 [driver dialect honeysql-form]
- ;; make sure [[driver/*driver*]] is bound, we need it for [[sqlize-value]]
- (binding [driver/*driver* driver]
- (sql/format honeysql-form {:dialect dialect
- :quoted true
- :quoted-snake false
- :inline driver/*compile-with-inline-parameters*
- ;; Enable :nested when we want to compile just one particular snippet.
- :nested (not (map? honeysql-form))})))
-
-(defmulti format-honeysql
- "Compile `honeysql-form` to a `[sql & args]` vector. Prior to 0.51.0, this was a plain function, but was made a
- multimethod in 0.51.0 to support drivers that need to always
- specify [[metabase.driver/*compile-with-inline-parameters*]]."
- {:arglists '([driver honeysql-form]), :added "0.51.0"}
- driver/dispatch-on-initialized-driver
- :hierarchy #'driver/hierarchy)
-
-(defmethod format-honeysql :sql
- [driver honeysql-form]
- (let [dialect (quote-style driver)]
- (try
- (format-honeysql-2 driver dialect honeysql-form)
- (catch Throwable e
- (try
- (log/error (u/format-color :red "Invalid HoneySQL form: %s" (ex-message e)))
- (finally
- (throw (ex-info (tru "Error compiling HoneySQL form: {0}" (ex-message e))
- {:dialect dialect
- :form honeysql-form
- :type driver-api/qp.error-type.driver}
- e))))))))
-
(defn- default-select [driver {[from] :from, :as _honeysql-form}]
(let [table-identifier (if (sequential? from)
;; Grab the alias part.
diff --git a/src/metabase/driver/sql/query_processor/format.clj b/src/metabase/driver/sql/query_processor/format.clj
new file mode 100644
index 00000000000..a541ee0f11a
--- /dev/null
+++ b/src/metabase/driver/sql/query_processor/format.clj
@@ -0,0 +1,64 @@
+(ns metabase.driver.sql.query-processor.format
+ "Compiling a Honey SQL form to a SQL string for a driver.
+
+ Drivers implement these as [[metabase.driver.sql.query-processor]] vars; that namespace re-exports them. They live
+ here because [[metabase.driver.sql.util/quote-name]] needs [[format-honeysql]], and `sql.util` is in turn used
+ by [[metabase.query-processor.util.persisted-cache]], which [[metabase.driver.sql.query-processor]] requires."
+ (:require
+ [honey.sql :as sql]
+ [metabase.driver :as driver]
+ [metabase.query-processor.error-type :as qp.error-type]
+ [metabase.util :as u]
+ [metabase.util.i18n :refer [tru]]
+ [metabase.util.log :as log]))
+
+(set! *warn-on-reflection* true)
+
+(defmulti quote-style
+ "Return the dialect that should be used by Honey SQL 2 when building a SQL statement. Defaults to `:ansi`, but other
+ valid options are `:mysql`, `:sqlserver`, `:oracle`, and `:h2` (added in
+ [[metabase.util.honey-sql-2]]; like `:ansi`, but uppercases the result). Check [[honey.sql/dialects]] for all
+ available dialects, or register a custom one with [[honey.sql/register-dialect!]].
+
+ (honey.sql/format ... :quoting (quote-style driver), :allow-dashed-names? true)
+
+ (The name of this method reflects Honey SQL 1 terminology, where \"dialect\" was called \"quote style\". To avoid
+ needless churn, I haven't changed it yet. -- Cam)"
+ {:added "0.32.0" :arglists '([driver])}
+ driver/dispatch-on-initialized-driver
+ :hierarchy #'driver/hierarchy)
+
+(defmethod quote-style :sql [_] :ansi)
+
+(defn- format-honeysql-2 [driver dialect honeysql-form]
+ ;; make sure [[driver/*driver*]] is bound, we need it for [[sqlize-value]]
+ (binding [driver/*driver* driver]
+ (sql/format honeysql-form {:dialect dialect
+ :quoted true
+ :quoted-snake false
+ :inline driver/*compile-with-inline-parameters*
+ ;; Enable :nested when we want to compile just one particular snippet.
+ :nested (not (map? honeysql-form))})))
+
+(defmulti format-honeysql
+ "Compile `honeysql-form` to a `[sql & args]` vector. Prior to 0.51.0, this was a plain function, but was made a
+ multimethod in 0.51.0 to support drivers that need to always
+ specify [[metabase.driver/*compile-with-inline-parameters*]]."
+ {:arglists '([driver honeysql-form]), :added "0.51.0"}
+ driver/dispatch-on-initialized-driver
+ :hierarchy #'driver/hierarchy)
+
+(defmethod format-honeysql :sql
+ [driver honeysql-form]
+ (let [dialect (quote-style driver)]
+ (try
+ (format-honeysql-2 driver dialect honeysql-form)
+ (catch Throwable e
+ (try
+ (log/error (u/format-color :red "Invalid HoneySQL form: %s" (ex-message e)))
+ (finally
+ (throw (ex-info (tru "Error compiling HoneySQL form: {0}" (ex-message e))
+ {:dialect dialect
+ :form honeysql-form
+ :type qp.error-type/driver}
+ e))))))))
diff --git a/src/metabase/driver/sql/util.clj b/src/metabase/driver/sql/util.clj
index 3ff65c09f43..d0cb4711a94 100644
--- a/src/metabase/driver/sql/util.clj
+++ b/src/metabase/driver/sql/util.clj
@@ -2,8 +2,8 @@
"Utility functions for writing SQL drivers."
(:require
[clojure.string :as str]
- [metabase.driver-api.core :as driver-api]
- [metabase.driver.sql.query-processor :as sql.qp]
+ [metabase.driver.sql.query-processor.format :as sql.qp.format]
+ [metabase.query-processor.error-type :as qp.error-type]
[metabase.util :as u]
[metabase.util.honey-sql-2 :as h2x]
[metabase.util.i18n :refer [tru]]
@@ -30,7 +30,7 @@
identifier-type :- h2x/IdentifierType
& components]
(first
- (sql.qp/format-honeysql driver (apply h2x/identifier identifier-type components))))
+ (sql.qp.format/format-honeysql driver (apply h2x/identifier identifier-type components))))
;;; +----------------------------------------------------------------------------------------------------------------+
;;; | Deduplicate Field Aliases |
@@ -205,12 +205,12 @@
[has-timezone? target-timezone source-timezone]
(when (and has-timezone? source-timezone)
(throw (ex-info (tru "input column already has a set timezone. Please remove the source parameter in convertTimezone.")
- {:type driver-api/qp.error-type.invalid-query
+ {:type qp.error-type/invalid-query
:target-timezone target-timezone
:source-timezone source-timezone})))
(when (and (not has-timezone?) (not source-timezone))
(throw (ex-info (tru "input column doesn''t have a set timezone. Please set the source parameter in convertTimezone to convert it.")
- {:type driver-api/qp.error-type.invalid-query
+ {:type qp.error-type/invalid-query
:target-timezone target-timezone
:source-timezone source-timezone}))))
diff --git a/src/metabase/driver/sql_jdbc/connection.clj b/src/metabase/driver/sql_jdbc/connection.clj
index f41c9a18962..8dd15d9b522 100644
--- a/src/metabase/driver/sql_jdbc/connection.clj
+++ b/src/metabase/driver/sql_jdbc/connection.clj
@@ -12,6 +12,7 @@
[metabase.driver.db :as driver.db]
[metabase.driver.settings :as driver.settings]
[metabase.driver.sql-jdbc.common :as sql-jdbc.common]
+ [metabase.driver.sql-jdbc.connection.pool-lock :as pool-lock]
[metabase.driver.sql-jdbc.connection.ssh-tunnel :as ssh]
[metabase.driver.util :as driver.u]
[metabase.util :as u]
@@ -476,7 +477,7 @@
;; We don't want to end up with a bunch of simultaneous threads creating pools only to have them destroyed
;; the very next instant. This will cause their queries to fail. Thus we should do the usual locking here
;; and make sure only one thread will be creating a pool at a given instant.
- (locking pool-cache-key->connection-pool
+ (locking pool-lock/monitor
(or
;; check if another thread created the pool while we were waiting to acquire the lock
(get-canonical-pool cache-key details-hash false)
diff --git a/src/metabase/driver/sql_jdbc/connection/pool_lock.clj b/src/metabase/driver/sql_jdbc/connection/pool_lock.clj
new file mode 100644
index 00000000000..5d5d03ff258
--- /dev/null
+++ b/src/metabase/driver/sql_jdbc/connection/pool_lock.clj
@@ -0,0 +1,12 @@
+(ns metabase.driver.sql-jdbc.connection.pool-lock
+ "The monitor serializing c3p0 pool creation against reads of c3p0's JMX attributes.
+
+ Holding it on both sides is required to avoid a deadlock inside c3p0: reading a JMX attribute locks a
+ DynamicPooledDataSourceManagerMBean and then a PoolBackedDataSource, while
+ `com.mchange.v2.c3p0.DataSources/pooledDataSource` locks the two in the opposite order. See
+ https://github.com/swaldman/c3p0/issues/95
+
+ Carries no requires so that the JMX reader can take the same monitor without loading the driver.")
+
+(defonce ^{:doc "The monitor object. Take it with `locking`."} monitor
+ (Object.))
diff --git a/src/metabase/driver/util/settings.clj b/src/metabase/driver/util/settings.clj
new file mode 100644
index 00000000000..b6433e71363
--- /dev/null
+++ b/src/metabase/driver/util/settings.clj
@@ -0,0 +1,20 @@
+(ns metabase.driver.util.settings
+ "The `engines` setting: which drivers this instance can use, and the connection form each one needs.
+
+ It lives here rather than alongside the other driver settings because its value comes from
+ [[metabase.driver.util/available-drivers-info]], while [[metabase.driver.util]] reads
+ [[metabase.driver.settings]] -- defining it there would make those two namespaces require each other.
+
+ Nothing refers to this namespace by name; the setting reaches the settings API only because
+ [[metabase.driver.init]] loads it."
+ (:require
+ [metabase.driver.util :as driver.util]
+ [metabase.settings.core :refer [defsetting]]))
+
+(defsetting engines
+ "Available database engines"
+ :encryption :no
+ :visibility :public
+ :setter :none
+ :getter driver.util/available-drivers-info
+ :doc false)
diff --git a/src/metabase/explorations/interestingness.clj b/src/metabase/explorations/interestingness.clj
index 0b6d6b2d2a6..36db4767b52 100644
--- a/src/metabase/explorations/interestingness.clj
+++ b/src/metabase/explorations/interestingness.clj
@@ -45,7 +45,7 @@
(extraction-units (lib/raw-temporal-bucket lib-col)))
(defn- start-of-week-day ^DayOfWeek []
- (-> ((requiring-resolve 'metabase.settings.core/get) :start-of-week)
+ (-> (lib-be/start-of-week)
(or :sunday)
name
(u/upper-case-en)
diff --git a/src/metabase/indexed_entities/models/model_index.clj b/src/metabase/indexed_entities/models/model_index.clj
index 8d669dc58b9..5f63f94085c 100644
--- a/src/metabase/indexed_entities/models/model_index.clj
+++ b/src/metabase/indexed_entities/models/model_index.clj
@@ -2,6 +2,7 @@
(:require
[clojure.set :as set]
[clojure.string :as str]
+ [clojurewerkz.quartzite.triggers :as triggers]
[metabase.indexed-entities.db :as indexed-entities.db]
;; legacy usage, do not use this in new code
^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.normalize :as mbql.normalize]
@@ -14,6 +15,7 @@
[metabase.query-processor.core :as qp]
[metabase.search.core :as search]
[metabase.sync.schedules :as sync.schedules]
+ [metabase.task.core :as task]
[metabase.util.cron :as u.cron]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
@@ -39,10 +41,22 @@
{:pk_ref mi/transform-legacy-field-ref
:value_ref mi/transform-legacy-field-ref})
+(defn trigger-key
+ "Quartz trigger key for the job that refreshes the values of model index `model-index-id`."
+ [model-index-id]
+ (triggers/key (format "metabase.task.IndexValues.trigger.%d" model-index-id)))
+
+(defn remove-indexing-job
+ "Remove the indexing job for `model-index`.
+
+ This and [[trigger-key]] live here rather than with the job itself so that deleting a ModelIndex can cancel its
+ trigger without reaching into [[metabase.indexed-entities.task.index-values]], which reads this namespace."
+ [model-index]
+ (task/delete-trigger! (trigger-key (:id model-index))))
+
(t2/define-before-delete :model/ModelIndex
[model-index]
- (let [remove-refresh-job (requiring-resolve 'metabase.indexed-entities.task.index-values/remove-indexing-job)]
- (remove-refresh-job model-index)))
+ (remove-indexing-job model-index))
(def max-indexed-values
"Maximum number of values we will index. Actually take one more than this to test if there are more than the
diff --git a/src/metabase/indexed_entities/task/index_values.clj b/src/metabase/indexed_entities/task/index_values.clj
index 9bc421c5645..e681a28e788 100644
--- a/src/metabase/indexed_entities/task/index_values.clj
+++ b/src/metabase/indexed_entities/task/index_values.clj
@@ -32,11 +32,6 @@
(not= (:type model) :model)
(:archived model)))
-(defn- model-index-trigger-key
- [model-index-id]
- (triggers/key
- (format "metabase.task.IndexValues.trigger.%d" model-index-id)))
-
(defn- refresh-index!
"Refresh the index on a model. Note, if the index should be removed (no longer a model, archived,
etc, (see [[should-deindex?]])) will delete the indexing job."
@@ -46,7 +41,7 @@
(indexed-entities.db/card (:model_id model-index)))]
(if (should-deindex? model model-index)
(u/ignore-exceptions
- (let [trigger-key (model-index-trigger-key model-index-id)]
+ (let [trigger-key (model-index/trigger-key model-index-id)]
(task/delete-trigger! trigger-key)
(indexed-entities.db/delete-model-index! model-index-id)))
(model-index/add-values! model-index))))
@@ -72,7 +67,7 @@
(defn- refresh-trigger ^org.quartz.CronTrigger [model-index]
(triggers/build
(triggers/with-description (format "Refresh index on model %d" (:model_id model-index)))
- (triggers/with-identity (model-index-trigger-key (:id model-index)))
+ (triggers/with-identity (model-index/trigger-key (:id model-index)))
(triggers/using-job-data {"model-index-id" (u/the-id model-index)})
(triggers/for-job (jobs/key refresh-model-index-key))
(triggers/start-now)
@@ -95,12 +90,6 @@
(catch Exception e
(log/warnf "Error scheduling indexing for model: %s: %s" (:model_id model-index) (ex-message e))))))
-(defn remove-indexing-job
- "Public API to remove an indexing job on a model."
- [model-index]
- (let [trigger-key (model-index-trigger-key (:id model-index))]
- (task/delete-trigger! trigger-key)))
-
(defn- recreate-missing-triggers!
"Ensure all model indexes in the database have triggers in Quartz."
[]
diff --git a/src/metabase/mcp/v2/projections.clj b/src/metabase/mcp/v2/projections.clj
index 87957b86ec0..47ef4f095cd 100644
--- a/src/metabase/mcp/v2/projections.clj
+++ b/src/metabase/mcp/v2/projections.clj
@@ -16,7 +16,7 @@
(:require
[clojure.string :as str]
[medley.core :as m]
- [metabase.dashboards.models.dashboard :as dashboard]
+ [metabase.parameters.params :as params]
[metabase.util :as u]))
(set! *warn-on-reflection* true)
@@ -252,9 +252,9 @@
(defn- dashboard-parameters-summary
"One row per dashboard parameter: id, name, type, and the ids of the dashcards it is wired
- to (via [[dashboard/dashboard->resolved-params]], the same resolution the QP uses)."
+ to (via [[params/dashboard->resolved-params]], the same resolution the QP uses)."
[dash]
- (let [resolved (dashboard/dashboard->resolved-params dash)]
+ (let [resolved (params/dashboard->resolved-params dash)]
(mapv (fn [{param-id :id :as param}]
(m/remove-vals
nil?
diff --git a/src/metabase/models/interface.clj b/src/metabase/models/interface.clj
index 5377db1f282..29f7f062e63 100644
--- a/src/metabase/models/interface.clj
+++ b/src/metabase/models/interface.clj
@@ -15,6 +15,7 @@
[clojure.string :as str]
[clojure.walk :as walk]
[medley.core :as m]
+ [metabase.app-db.core :as mdb]
;; Toucan out-transforms normalize stored legacy MBQL on read; needed until the app db is MBQL 5
^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.normalize :as mbql.normalize]
;; stored card queries/refs are still legacy MBQL; validated against the legacy schema on read/write
@@ -552,7 +553,7 @@
and H2 and `now(6)` for MySQL/MariaDB (`now()` for MySQL only return second resolution; `now(6)` uses the
max (nanosecond) resolution)."
[]
- (h2x/current-datetime-honeysql-form ((requiring-resolve 'metabase.app-db.core/db-type))))
+ (h2x/current-datetime-honeysql-form (mdb/db-type)))
(defn- add-created-at-timestamp [obj & _]
(cond-> obj
diff --git a/src/metabase/models/serialization.clj b/src/metabase/models/serialization.clj
index c1353901a60..7dae9ebdd32 100644
--- a/src/metabase/models/serialization.clj
+++ b/src/metabase/models/serialization.clj
@@ -70,7 +70,9 @@
[metabase.lib.schema.parameter :as lib.schema.parameter]
[metabase.models.db :as models.db]
[metabase.models.interface :as mi]
+ [metabase.models.serialization.path]
[metabase.models.serialization.resolve :as resolve]
+ [metabase.models.serialization.resolve.default :as resolve.default]
[metabase.models.visualization-settings :as mb.viz]
[metabase.util :as u]
[metabase.util.date-2 :as u.date]
@@ -80,12 +82,22 @@
[metabase.util.malli.humanize :as mu.humanize]
[metabase.util.malli.registry :as mr]
[metabase.util.match :as match]
+ [potemkin :as p]
[toucan2.core :as t2]
[toucan2.model :as t2.model]
[toucan2.realize :as t2.realize]))
(set! *warn-on-reflection* true)
+(p/import-vars
+ [metabase.models.serialization.path
+ entity-id
+ field-hierarchy
+ generate-path
+ infer-self-path
+ lookup-by-id
+ maybe-labeled])
+
;; there was no science behind picking 100 as a number
(def ^:private extract-nested-batch-limit "max amount of entities to fetch nested entities for" 100)
@@ -93,17 +105,6 @@
"Maximum number of ids per `:in` clause, to stay under database parameter limits."
1000)
-(mr/def ::model-keyword
- [:and
- qualified-keyword?
- [:fn
- {:error/message ":model/X keyword"}
- #(when (qualified-keyword? %)
- (= (namespace %) "model"))]])
-
-(mr/def ::model-keyword-or-symbol
- [:or symbol? ::model-keyword])
-
;;; # Serialization Overview
;;;
;;; Serialization (or "serdes") is a system for exporting entities (Dashboards, Cards, Collections, etc.) from one
@@ -142,19 +143,6 @@
;;; - For entities that existed before the column was added, have a portable way to rebuild them (see below on
;;; hashing).
-(defmulti entity-id
- "Given the model name and an entity, returns its entity ID (which might be nil).
-
- This abstracts over the exact definition of the \"entity ID\" for a given entity.
- By default this is a column, `:entity_id`.
-
- Models that have a different portable ID (`Database`, `Field`, etc.) should override this."
- {:arglists '([model-name instance])}
- (fn [model-name _instance] model-name))
-
-(defmethod entity-id :default [_ instance]
- (some-> instance :entity_id str/trim))
-
(defn has-entity-id?
"Returns true if the model has an `:entity_id` column."
[model]
@@ -200,41 +188,6 @@
;;; ## Two kinds of nesting
;;; To reiterate, `:serdes/meta` paths are not filesystem paths. When `extract`ed entities are stored to disk.
-(defmulti generate-path
- "Given the model name and raw entity from the database, returns a vector giving its *path*.
- `(generate-path \"ModelName\" entity)`
-
- The path is a vector of maps, root first and this entity itself last. Each map looks like:
- `{:model \"ModelName\" :id \"entity ID, identity hash, or custom ID\" :label \"optional human label\"}`
-
- Nested models with no entity_id need to return nil for generate-path."
- {:arglists '([model-name instance])}
- (fn [model-name _instance] model-name))
-
-(defn infer-self-path
- "Returns `{:model \"ModelName\" :id \"id-string\"}`"
- [model-name entity]
- {:model model-name
- :id (entity-id model-name entity)})
-
-(defn maybe-labeled
- "Common helper for defining [[generate-path]] for an entity that is
- (1) top-level, ie. a one layer path;
- (2) labeled by a single field, slugified.
-
- For example, a Card's or Dashboard's `:name` field."
- [model-name entity slug-key]
- (let [self (infer-self-path model-name entity)
- label (slug-key entity)]
- [(-> self
- (m/assoc-some :label (some-> label (u/slugify {:unicode? true}))))]))
-
-(defmethod generate-path :default [model-name entity]
- ;; This default works for most models, but needs overriding for those that don't rely on entity_id.
- (maybe-labeled model-name entity #(if (string? (:name %))
- (:name %)
- (:format-string (:name %)))))
-
(defn log-path-str
"Returns a string for logging from a serdes path sequence (i.e. in :serdes/meta)"
[elements]
@@ -643,8 +596,6 @@
(fn [path]
(-> path last :model)))
-(declare lookup-by-id)
-
(defmethod load-find-local :default [path]
(let [{id :id model-name :model} (last path)
model (t2.model/resolve-model (symbol model-name))]
@@ -799,12 +750,6 @@
[id-str]
(resolve/entity-id? id-str))
-(mu/defn lookup-by-id
- "Given an entity ID string, finds the matching entity. This is useful when writing [[xform-one]] to
- turn a foreign key from a portable form to an appdb ID. Returns a Toucan entity or nil."
- [model :- ::model-keyword-or-symbol id-str]
- (models.db/entity-by-entity-id model id-str))
-
(defn storage-default-collection-path
"Implements the most common structure for [[storage-path]].
Returns a vector of maps with `:label` and `:key` for each path segment.
@@ -861,17 +806,13 @@
;;; These wrapper functions delegate to the current resolver (set by [[with-cache]]).
;;; When no resolver is bound, they fall back to the database-backed resolver.
-;; TODO: `requiring-resolve` is needed here because resolve.default requires this ns
-;; (for `generate-path`, `field-hierarchy`, `lookup-by-id`).
-;; Moving those into resolve.default (or a shared utils ns) would break the cycle and
-;; let us require resolve.default directly.
(defn- export-resolver []
(or resolve/*export-resolver*
- @(requiring-resolve 'metabase.models.serialization.resolve.default/default-export-resolver)))
+ resolve.default/default-export-resolver))
(defn- import-resolver []
(or resolve/*import-resolver*
- @(requiring-resolve 'metabase.models.serialization.resolve.default/default-import-resolver)))
+ resolve.default/default-import-resolver))
;;; ## General foreign keys
@@ -883,7 +824,7 @@
NOTE: This works for both top-level and nested entities. Top-level entities like `Card` are returned as just a
portable ID string.. Nested entities are returned as a vector of such ID strings."
[id :- [:maybe pos-int?]
- model :- ::model-keyword-or-symbol]
+ model :- :metabase.models.serialization.path/model-keyword-or-symbol]
(resolve/export-fk (export-resolver) id model))
(defmacro ^:private fk-elide
@@ -909,7 +850,7 @@
Unusual parameter order means this can be used as `(update x :some_id import-fk 'SomeModel)`."
[eid
- model :- ::model-keyword-or-symbol]
+ model :- :metabase.models.serialization.path/model-keyword-or-symbol]
(resolve/import-fk (import-resolver) eid model))
(mu/defn ^:dynamic *export-fk-keyed*
@@ -920,7 +861,7 @@
Note: This assumes the primary key is called `:id`."
[id
- model :- ::model-keyword-or-symbol
+ model :- :metabase.models.serialization.path/model-keyword-or-symbol
field]
(resolve/export-fk-keyed (export-resolver) id model field))
@@ -1019,12 +960,6 @@
;;; ## Fields
-(defn field-hierarchy
- "Returns the field hierarchy (field + parents) for a field ID. Used by resolvers."
- [id]
- (reverse
- (models.db/field-hierarchy-rows id)))
-
;; NOTE: field lookups are intentionally NOT routed through the cached resolver, unlike the
;; database and table exporters above. Fields are unbounded in number (millions on large
;; instances), and the dominant traffic — each field's own path during a data-model export —
@@ -1784,7 +1719,7 @@
left in its portable entity-id form, instead of throwing. Use for paths where a deleted source Card
should be treated as a broken section rather than break the whole read."
[settings]
- (binding [resolve/*import-resolver* @(requiring-resolve 'metabase.models.serialization.resolve.default/lenient-import-resolver)]
+ (binding [resolve/*import-resolver* resolve.default/lenient-import-resolver]
(import-visualizer-settings settings)))
(defn import-visualization-settings
@@ -1980,6 +1915,6 @@
(defmacro with-cache
"Runs body with resolvers bound to cached (memoized) versions for performance."
[& body]
- `(binding [resolve/*export-resolver* ((requiring-resolve 'metabase.models.serialization.resolve.default/cached-export-resolver))
- resolve/*import-resolver* ((requiring-resolve 'metabase.models.serialization.resolve.default/cached-import-resolver))]
+ `(binding [resolve/*export-resolver* (resolve.default/cached-export-resolver)
+ resolve/*import-resolver* (resolve.default/cached-import-resolver)]
~@body))
diff --git a/src/metabase/models/serialization/path.clj b/src/metabase/models/serialization/path.clj
new file mode 100644
index 00000000000..785df762d75
--- /dev/null
+++ b/src/metabase/models/serialization/path.clj
@@ -0,0 +1,86 @@
+(ns metabase.models.serialization.path
+ "Portable identity for serdes entities: entity IDs, the `:serdes/meta` path built from them, and the appdb lookups
+ that go the other way.
+
+ Lives below [[metabase.models.serialization]] so that resolver implementations can use it without depending on the
+ serdes interface, which depends on them in turn."
+ (:require
+ [clojure.string :as str]
+ [medley.core :as m]
+ [metabase.models.db :as models.db]
+ [metabase.util :as u]
+ [metabase.util.malli :as mu]
+ [metabase.util.malli.registry :as mr]))
+
+(set! *warn-on-reflection* true)
+
+(mr/def ::model-keyword
+ [:and
+ qualified-keyword?
+ [:fn
+ {:error/message ":model/X keyword"}
+ #(when (qualified-keyword? %)
+ (= (namespace %) "model"))]])
+
+(mr/def ::model-keyword-or-symbol
+ [:or symbol? ::model-keyword])
+
+(defmulti entity-id
+ "Given the model name and an entity, returns its entity ID (which might be nil).
+
+ This abstracts over the exact definition of the \"entity ID\" for a given entity.
+ By default this is a column, `:entity_id`.
+
+ Models that have a different portable ID (`Database`, `Field`, etc.) should override this."
+ {:arglists '([model-name instance])}
+ (fn [model-name _instance] model-name))
+
+(defmethod entity-id :default [_ instance]
+ (some-> instance :entity_id str/trim))
+
+(defmulti generate-path
+ "Given the model name and raw entity from the database, returns a vector giving its *path*.
+ `(generate-path \"ModelName\" entity)`
+
+ The path is a vector of maps, root first and this entity itself last. Each map looks like:
+ `{:model \"ModelName\" :id \"entity ID, identity hash, or custom ID\" :label \"optional human label\"}`
+
+ Nested models with no entity_id need to return nil for generate-path."
+ {:arglists '([model-name instance])}
+ (fn [model-name _instance] model-name))
+
+(defn infer-self-path
+ "Returns `{:model \"ModelName\" :id \"id-string\"}`"
+ [model-name entity]
+ {:model model-name
+ :id (entity-id model-name entity)})
+
+(defn maybe-labeled
+ "Common helper for defining [[generate-path]] for an entity that is
+ (1) top-level, ie. a one layer path;
+ (2) labeled by a single field, slugified.
+
+ For example, a Card's or Dashboard's `:name` field."
+ [model-name entity slug-key]
+ (let [self (infer-self-path model-name entity)
+ label (slug-key entity)]
+ [(-> self
+ (m/assoc-some :label (some-> label (u/slugify {:unicode? true}))))]))
+
+(defmethod generate-path :default [model-name entity]
+ ;; This default works for most models, but needs overriding for those that don't rely on entity_id.
+ (maybe-labeled model-name entity #(if (string? (:name %))
+ (:name %)
+ (:format-string (:name %)))))
+
+(mu/defn lookup-by-id
+ "Given an entity ID string, finds the matching entity. This is useful when writing [[metabase.models.serialization/xform-one]] to
+ turn a foreign key from a portable form to an appdb ID. Returns a Toucan entity or nil."
+ [model :- ::model-keyword-or-symbol id-str]
+ (models.db/entity-by-entity-id model id-str))
+
+(defn field-hierarchy
+ "Returns the field hierarchy (field + parents) for a field ID. Used by resolvers."
+ [id]
+ (reverse
+ (models.db/field-hierarchy-rows id)))
diff --git a/src/metabase/models/serialization/resolve/default.clj b/src/metabase/models/serialization/resolve/default.clj
index 7324924fe0c..86960b40944 100644
--- a/src/metabase/models/serialization/resolve/default.clj
+++ b/src/metabase/models/serialization/resolve/default.clj
@@ -5,7 +5,7 @@
against the application database."
(:require
[metabase.models.db :as models.db]
- [metabase.models.serialization :as serdes]
+ [metabase.models.serialization.path :as serdes.path]
[metabase.models.serialization.resolve :as resolve]
[metabase.util.log :as log]
[toucan2.core :as t2])
@@ -24,7 +24,7 @@
(let [model-name (name model)
entity (models.db/entity-by-own-pk model id)
path (when entity
- (mapv :id (serdes/generate-path model-name entity)))]
+ (mapv :id (serdes.path/generate-path model-name entity)))]
(cond
(nil? entity) (throw (ex-info "FK target not found" {:model model
:id id
@@ -55,7 +55,7 @@
"Given a numeric field_id, return a portable field reference [db-name schema table-name field-name]."
[resolver field-id]
(when field-id
- (let [fields (serdes/field-hierarchy field-id)
+ (let [fields (serdes.path/field-hierarchy field-id)
[db-name schema field-name] (resolve/export-table-fk resolver (:table_id (first fields)))]
(into [db-name schema field-name] (map :name fields)))))
@@ -68,7 +68,7 @@
[eid model]
(when eid
(let [eid (if (vector? eid) (last eid) eid)
- entity (serdes/lookup-by-id model eid)]
+ entity (serdes.path/lookup-by-id model eid)]
(if entity
(get entity (first (t2/primary-keys model)))
(throw (ex-info "Could not find foreign key target - bad serdes dependencies or other serialization error"
diff --git a/src/metabase/notification/events/report_timezone_updated.clj b/src/metabase/notification/events/report_timezone_updated.clj
index 07baa71998c..60e4ce35a5f 100644
--- a/src/metabase/notification/events/report_timezone_updated.clj
+++ b/src/metabase/notification/events/report_timezone_updated.clj
@@ -1,7 +1,7 @@
(ns metabase.notification.events.report-timezone-updated
(:require
[metabase.events.core :as events]
- [metabase.notification.task.send :as send]
+ [metabase.notification.task.send-trigger :as notification.task.send-trigger]
[methodical.core :as methodical]))
(events/derive! ::event :metabase/event)
@@ -10,4 +10,4 @@
(methodical/defmethod events/publish-event! ::event
"When the report-timezone Setting is updated, update the timezone of all SendNotification triggers."
[_topic _event]
- (send/update-send-notification-triggers-timezone!))
+ (notification.task.send-trigger/update-send-notification-triggers-timezone!))
diff --git a/src/metabase/notification/models.clj b/src/metabase/notification/models.clj
index f314fe90c52..3c0c7547dcf 100644
--- a/src/metabase/notification/models.clj
+++ b/src/metabase/notification/models.clj
@@ -11,6 +11,7 @@
[metabase.models.interface :as mi]
[metabase.models.util.spec-update :as models.u.spec-update]
[metabase.notification.db :as notification.db]
+ [metabase.notification.task.send-trigger :as notification.task.send-trigger]
[metabase.permissions.core :as perms]
[metabase.permissions.schema :as permissions.schema]
[metabase.premium-features.core :as premium-features :refer [defenterprise]]
@@ -148,14 +149,6 @@
(validate-notification instance)
instance)
-(defn- update-subscription-trigger!
- [& args]
- (apply (requiring-resolve 'metabase.notification.task.send/update-subscription-trigger!) args))
-
-(defn- delete-trigger-for-subscription!
- [& args]
- (apply (requiring-resolve 'metabase.notification.task.send/delete-trigger-for-subscription!) args))
-
(t2/define-before-update :model/Notification
[instance]
(validate-notification instance)
@@ -175,14 +168,14 @@
(let [subscriptions (notification.db/cron-subscriptions-for-notification (:id instance))]
(doseq [subscription subscriptions]
(if (:active instance)
- (update-subscription-trigger! subscription)
- (delete-trigger-for-subscription! (:id subscription))))))
+ (notification.task.send-trigger/update-subscription-trigger! subscription)
+ (notification.task.send-trigger/delete-trigger-for-subscription! (:id subscription))))))
instance)
(t2/define-before-delete :model/Notification
[instance]
(doseq [subscription-id (notification.db/cron-subscription-ids-for-notification (:id instance))]
- (delete-trigger-for-subscription! subscription-id))
+ (notification.task.send-trigger/delete-trigger-for-subscription! subscription-id))
(when-let [payload-id (:payload_id instance)]
(case (:payload_type instance)
:notification/card (notification.db/delete-notification-card! payload-id)))
@@ -248,18 +241,18 @@
(t2/define-after-insert :model/NotificationSubscription
[instance]
- (update-subscription-trigger! instance)
+ (notification.task.send-trigger/update-subscription-trigger! instance)
instance)
(t2/define-before-update :model/NotificationSubscription
[instance]
(validate-subscription instance)
- (update-subscription-trigger! instance)
+ (notification.task.send-trigger/update-subscription-trigger! instance)
instance)
(t2/define-before-delete :model/NotificationSubscription
[instance]
- (delete-trigger-for-subscription! (:id instance))
+ (notification.task.send-trigger/delete-trigger-for-subscription! (:id instance))
instance)
;; ------------------------------------------------------------------------------------------------;;
diff --git a/src/metabase/notification/task/send.clj b/src/metabase/notification/task/send.clj
index ef7caf94908..1ed0c0e73ea 100644
--- a/src/metabase/notification/task/send.clj
+++ b/src/metabase/notification/task/send.clj
@@ -1,103 +1,20 @@
(ns metabase.notification.task.send
(:require
- [clojure.data :refer [diff]]
[clojurewerkz.quartzite.conversion :as qc]
[clojurewerkz.quartzite.jobs :as jobs]
- [clojurewerkz.quartzite.schedule.cron :as cron]
[clojurewerkz.quartzite.triggers :as triggers]
- [metabase.driver :as driver]
[metabase.notification.db :as notification.db]
[metabase.notification.send :as notification.send]
- [metabase.query-processor.timezone :as qp.timezone]
+ [metabase.notification.task.send-trigger :as notification.task.send-trigger]
[metabase.task-history.core :as task-history]
[metabase.task.core :as task]
[metabase.tracing.core :as tracing]
[metabase.util.log :as log])
(:import
- (java.util TimeZone)
- (org.quartz CronTrigger DisallowConcurrentExecution JobExecutionContext TriggerKey)))
+ (org.quartz DisallowConcurrentExecution JobExecutionContext)))
(set! *warn-on-reflection* true)
-(def ^:private send-notification-job-key (jobs/key "metabase.task.notification.send.job"))
-
-(defn- send-notification-timezone
- []
- (or (driver/report-timezone)
- (qp.timezone/system-timezone-id)
- "UTC"))
-
-(defn- send-notification-trigger-key
- ^TriggerKey [subscription-id]
- (triggers/key (format "metabase.task.notification.trigger.subscription.%d"
- subscription-id)))
-
-(defn- send-notification-trigger-key->subscription-id
- [trigger-key]
- (when-let [[_ m] (re-matches #"metabase\.task\.notification\.trigger\.subscription\.(\d+)" (str trigger-key))]
- (parse-long m)))
-
-(defn- build-trigger
- "Build a Quartz Trigger for `database` and `task-info` if a schedule exists."
- ^CronTrigger [subscription-id cron-schedule]
- (triggers/build
- (triggers/with-description (format "Notification Subscription %d" subscription-id))
- (triggers/with-identity (send-notification-trigger-key subscription-id))
- (triggers/using-job-data {"subscription-id" subscription-id})
- (triggers/for-job send-notification-job-key)
- (triggers/start-now)
- (triggers/with-schedule
- (cron/schedule
- (cron/cron-schedule cron-schedule)
- (cron/in-time-zone (TimeZone/getTimeZone ^String (send-notification-timezone)))
- ;; We want to fire the trigger once even if the previous triggers missed
- (cron/with-misfire-handling-instruction-fire-and-proceed)))
- ;; higher than sync
- (triggers/with-priority 6)))
-
-(defn- create-new-trigger!
- [{:keys [id cron_schedule] :as _notification-subscription}]
- (task/add-trigger! (build-trigger id cron_schedule)))
-
-(defn update-subscription-trigger!
- "Update the trigger for a notification subscription if it exists and needs to be updated."
- [{:keys [id type cron_schedule] :as notification-subscription}]
- (let [existing-trigger (first (task/existing-triggers send-notification-job-key (send-notification-trigger-key id)))]
- (cond
- ;; delete trigger if type changes
- (and
- (not= type :notification-subscription/cron)
- existing-trigger)
- (do
- (log/infof "Deleting trigger for subscription %d because of type changes" id)
- (task/delete-trigger! (-> existing-trigger :key triggers/key)))
-
- ;; do nothing if type is not cron
- (not= type :notification-subscription/cron)
- nil
-
- ;; create new if there is no existing trigger
- (not existing-trigger)
- (do
- (log/infof "Creating new trigger for subscription %d with schedule %s" id cron_schedule)
- (create-new-trigger! notification-subscription))
-
- (not= cron_schedule (:schedule existing-trigger))
- (do
- (log/infof "Rescheduling trigger for subscription %d from %s to %s" id (:schedule existing-trigger) cron_schedule)
- (task/delete-trigger! (-> existing-trigger :key triggers/key))
- (create-new-trigger! notification-subscription))
-
- :else
- (log/infof "No changes to trigger for subscription %d" id))))
-
-(defn delete-trigger-for-subscription!
- "Delete the trigger for a notification subscription."
- [notification-subscription-id]
- (when-first [trigger (task/existing-triggers send-notification-job-key (send-notification-trigger-key notification-subscription-id))]
- (log/infof "Deleting trigger for subscription %d" notification-subscription-id)
- (task/delete-trigger! (-> trigger :key triggers/key))))
-
(defn- send-notification*
[subscription-id]
(let [subscription (notification.db/subscription subscription-id)
@@ -126,29 +43,12 @@
(nil? notification)
(do
(log/warnf "Skipping and deleting trigger for subscription %d because it does not exist." subscription-id)
- (delete-trigger-for-subscription! subscription-id))
+ (notification.task.send-trigger/delete-trigger-for-subscription! subscription-id))
(not (:active notification))
(do
(log/warnf "Skipping and deleting trigger for subscription %d because the notification is deactivated" subscription-id)
- (delete-trigger-for-subscription! subscription-id))))))
-
-(defn- active-cron-subscription-id->subscription
- []
- (notification.db/active-cron-subscriptions-by-id))
-
-(defn update-send-notification-triggers-timezone!
- "Update the timezone of all SendNotification triggers if the report timezone changes."
- []
- (let [triggers (-> send-notification-job-key task/job-info :triggers)
- new-timezone (send-notification-timezone)
- subscription-id->cron (update-vals (active-cron-subscription-id->subscription) :cron_schedule)]
- (doseq [trigger triggers
- :when (not= new-timezone (:timezone trigger))] ; skip if timezone is the same
- (let [trigger-key (:key trigger)
- subscription-id (send-notification-trigger-key->subscription-id trigger-key)]
- (log/infof "Updating timezone of trigger %s to %s. Was: %s" trigger-key new-timezone (:timezone trigger))
- (task/reschedule-trigger! (build-trigger subscription-id (get subscription-id->cron subscription-id)))))))
+ (notification.task.send-trigger/delete-trigger-for-subscription! subscription-id))))))
(task/defjob ^{:doc "Triggers that send a notification for a subscription."}
SendNotification
@@ -173,24 +73,6 @@
subscription-id trigger-key scheduled-fire-time fire-time recovering? refire-count scheduler-id)
(send-notification* subscription-id))))
-(defn init-send-notification-triggers!
- "Initialize all notification subscription triggers.
-
- Called when starting the instance."
- []
- (assert (task/scheduler) "Scheduler must be started before initializing SendNotification triggers")
-
- ;; Get all existing triggers and subscription IDs
- (let [existing-triggers (:triggers (task/job-info send-notification-job-key))
- existing-triggers-subscription-ids (map #(get-in % [:data "subscription-id"]) existing-triggers)
- subscription-id->subscription (active-cron-subscription-id->subscription)
- db-subscription-ids (keys subscription-id->subscription)
- [to-delete to-create _to-skip] (diff (set existing-triggers-subscription-ids) (set db-subscription-ids))]
- (doseq [subscription-id to-delete]
- (delete-trigger-for-subscription! subscription-id))
- (doseq [subscription-id to-create]
- (create-new-trigger! (get subscription-id->subscription subscription-id)))))
-
(task/defjob
^{:doc
"Find all notification subscriptions with cron schedules and create a trigger for each.
@@ -204,11 +86,11 @@
InitNotificationTriggers
[_context]
(log/info "Initializing SendNotification triggers")
- (init-send-notification-triggers!))
+ (notification.task.send-trigger/init-send-notification-triggers!))
(defmethod task/init! ::SendNotifications [_]
(let [send-notification-job (jobs/build
- (jobs/with-identity send-notification-job-key)
+ (jobs/with-identity notification.task.send-trigger/send-notification-job-key)
(jobs/with-description "Send Notification")
(jobs/of-type SendNotification)
(jobs/store-durably))
diff --git a/src/metabase/notification/task/send_trigger.clj b/src/metabase/notification/task/send_trigger.clj
new file mode 100644
index 00000000000..88101cdc4e7
--- /dev/null
+++ b/src/metabase/notification/task/send_trigger.clj
@@ -0,0 +1,136 @@
+(ns metabase.notification.task.send-trigger
+ "Quartz trigger lifecycle for notification subscriptions.
+
+ Separate from [[metabase.notification.task.send]] so that Notification model hooks can (un)schedule triggers: the job
+ body over there loads the sending machinery, which reads the models."
+ (:require
+ [clojure.data :refer [diff]]
+ [clojurewerkz.quartzite.jobs :as jobs]
+ [clojurewerkz.quartzite.schedule.cron :as cron]
+ [clojurewerkz.quartzite.triggers :as triggers]
+ [metabase.driver :as driver]
+ [metabase.notification.db :as notification.db]
+ [metabase.query-processor.timezone :as qp.timezone]
+ [metabase.task.core :as task]
+ [metabase.util.log :as log])
+ (:import
+ (java.util TimeZone)
+ (org.quartz CronTrigger TriggerKey)))
+
+(set! *warn-on-reflection* true)
+
+(def send-notification-job-key
+ "Key of the Quartz job that sends a notification when a subscription's trigger fires."
+ (jobs/key "metabase.task.notification.send.job"))
+
+(defn- send-notification-timezone
+ []
+ (or (driver/report-timezone)
+ (qp.timezone/system-timezone-id)
+ "UTC"))
+
+(defn- send-notification-trigger-key
+ ^TriggerKey [subscription-id]
+ (triggers/key (format "metabase.task.notification.trigger.subscription.%d"
+ subscription-id)))
+
+(defn- send-notification-trigger-key->subscription-id
+ [trigger-key]
+ (when-let [[_ m] (re-matches #"metabase\.task\.notification\.trigger\.subscription\.(\d+)" (str trigger-key))]
+ (parse-long m)))
+
+(defn- build-trigger
+ "Build a Quartz Trigger for `database` and `task-info` if a schedule exists."
+ ^CronTrigger [subscription-id cron-schedule]
+ (triggers/build
+ (triggers/with-description (format "Notification Subscription %d" subscription-id))
+ (triggers/with-identity (send-notification-trigger-key subscription-id))
+ (triggers/using-job-data {"subscription-id" subscription-id})
+ (triggers/for-job send-notification-job-key)
+ (triggers/start-now)
+ (triggers/with-schedule
+ (cron/schedule
+ (cron/cron-schedule cron-schedule)
+ (cron/in-time-zone (TimeZone/getTimeZone ^String (send-notification-timezone)))
+ ;; We want to fire the trigger once even if the previous triggers missed
+ (cron/with-misfire-handling-instruction-fire-and-proceed)))
+ ;; higher than sync
+ (triggers/with-priority 6)))
+
+(defn- create-new-trigger!
+ [{:keys [id cron_schedule] :as _notification-subscription}]
+ (task/add-trigger! (build-trigger id cron_schedule)))
+
+(defn update-subscription-trigger!
+ "Update the trigger for a notification subscription if it exists and needs to be updated."
+ [{:keys [id type cron_schedule] :as notification-subscription}]
+ (let [existing-trigger (first (task/existing-triggers send-notification-job-key (send-notification-trigger-key id)))]
+ (cond
+ ;; delete trigger if type changes
+ (and
+ (not= type :notification-subscription/cron)
+ existing-trigger)
+ (do
+ (log/infof "Deleting trigger for subscription %d because of type changes" id)
+ (task/delete-trigger! (-> existing-trigger :key triggers/key)))
+
+ ;; do nothing if type is not cron
+ (not= type :notification-subscription/cron)
+ nil
+
+ ;; create new if there is no existing trigger
+ (not existing-trigger)
+ (do
+ (log/infof "Creating new trigger for subscription %d with schedule %s" id cron_schedule)
+ (create-new-trigger! notification-subscription))
+
+ (not= cron_schedule (:schedule existing-trigger))
+ (do
+ (log/infof "Rescheduling trigger for subscription %d from %s to %s" id (:schedule existing-trigger) cron_schedule)
+ (task/delete-trigger! (-> existing-trigger :key triggers/key))
+ (create-new-trigger! notification-subscription))
+
+ :else
+ (log/infof "No changes to trigger for subscription %d" id))))
+
+(defn delete-trigger-for-subscription!
+ "Delete the trigger for a notification subscription."
+ [notification-subscription-id]
+ (when-first [trigger (task/existing-triggers send-notification-job-key (send-notification-trigger-key notification-subscription-id))]
+ (log/infof "Deleting trigger for subscription %d" notification-subscription-id)
+ (task/delete-trigger! (-> trigger :key triggers/key))))
+
+(defn- active-cron-subscription-id->subscription
+ []
+ (notification.db/active-cron-subscriptions-by-id))
+
+(defn update-send-notification-triggers-timezone!
+ "Update the timezone of all SendNotification triggers if the report timezone changes."
+ []
+ (let [triggers (-> send-notification-job-key task/job-info :triggers)
+ new-timezone (send-notification-timezone)
+ subscription-id->cron (update-vals (active-cron-subscription-id->subscription) :cron_schedule)]
+ (doseq [trigger triggers
+ :when (not= new-timezone (:timezone trigger))] ; skip if timezone is the same
+ (let [trigger-key (:key trigger)
+ subscription-id (send-notification-trigger-key->subscription-id trigger-key)]
+ (log/infof "Updating timezone of trigger %s to %s. Was: %s" trigger-key new-timezone (:timezone trigger))
+ (task/reschedule-trigger! (build-trigger subscription-id (get subscription-id->cron subscription-id)))))))
+
+(defn init-send-notification-triggers!
+ "Initialize all notification subscription triggers.
+
+ Called when starting the instance."
+ []
+ (assert (task/scheduler) "Scheduler must be started before initializing SendNotification triggers")
+
+ ;; Get all existing triggers and subscription IDs
+ (let [existing-triggers (:triggers (task/job-info send-notification-job-key))
+ existing-triggers-subscription-ids (map #(get-in % [:data "subscription-id"]) existing-triggers)
+ subscription-id->subscription (active-cron-subscription-id->subscription)
+ db-subscription-ids (keys subscription-id->subscription)
+ [to-delete to-create _to-skip] (diff (set existing-triggers-subscription-ids) (set db-subscription-ids))]
+ (doseq [subscription-id to-delete]
+ (delete-trigger-for-subscription! subscription-id))
+ (doseq [subscription-id to-create]
+ (create-new-trigger! (get subscription-id->subscription subscription-id)))))
diff --git a/src/metabase/parameters/params.clj b/src/metabase/parameters/params.clj
index 04ba37baab0..20f67bcbeec 100644
--- a/src/metabase/parameters/params.clj
+++ b/src/metabase/parameters/params.clj
@@ -27,6 +27,7 @@
[metabase.util.log :as log]
[metabase.util.malli :as mu]
[metabase.util.malli.registry :as mr]
+ [metabase.util.malli.schema :as ms]
[methodical.core :as methodical]
[toucan2.core :as t2]))
@@ -471,3 +472,27 @@
To get these IDs broken out by the Param ID that references them, use [[card->template-tag-param-id->field-ids]]."
[card :- [:maybe :map]]
(some-> card :dataset_query not-empty lib-be/normalize-query lib/all-template-tag-field-ids not-empty))
+
+(def ^:private ParamWithMapping
+ [:map
+ [:id ms/NonBlankString]
+ [:name ms/NonBlankString]
+ [:mappings [:maybe [:set ::parameters.schema/parameter-mapping-with-dashcard]]]])
+
+;; Kept out of [[metabase.dashboards.models.dashboard]] so the query processor can use it without
+;; depending on the Dashboard model.
+(mu/defn dashboard->resolved-params :- [:map-of ms/NonBlankString ParamWithMapping]
+ "Return map of Dashboard parameter key -> param with resolved `:mappings` (see the `:resolved-params` hydration
+ in [[metabase.dashboards.models.dashboard]] for an example). Callers that only need the mappings (e.g. the QP) can
+ pass slim dashcards instead of paying for the full hydration."
+ [dashboard :- [:map
+ [:parameters [:maybe [:sequential :map]]]
+ [:dashcards [:maybe [:sequential [:map
+ [:parameter_mappings [:maybe [:sequential :map]]]]]]]]]
+ (let [param-key->mappings (apply
+ merge-with set/union
+ (for [dashcard (:dashcards dashboard)
+ param (:parameter_mappings dashcard)]
+ {(:parameter_id param) #{(assoc param :dashcard dashcard)}}))]
+ (into {} (for [{param-key :id, :as param} (:parameters dashboard)]
+ [(u/qualified-name param-key) (assoc param :mappings (get param-key->mappings param-key))]))))
diff --git a/src/metabase/permissions/models/collection/graph.clj b/src/metabase/permissions/models/collection/graph.clj
index e70f0b5d937..1ac014f4dca 100644
--- a/src/metabase/permissions/models/collection/graph.clj
+++ b/src/metabase/permissions/models/collection/graph.clj
@@ -6,6 +6,7 @@
[clojure.set :as set]
[metabase.api.common :as api]
[metabase.audit-app.core :as audit]
+ [metabase.collections.models.collection.root :as collection.root]
[metabase.permissions.db :as permissions.db]
[metabase.permissions.models.collection-permission-graph-revision :as c-perm-revision]
[metabase.permissions.models.permissions :as perms]
@@ -51,11 +52,6 @@
;;; -------------------------------------------------- Fetch Graph ---------------------------------------------------
;;;
-(defn- root-collection
- "Requiring resolve to break a circular dependency"
- []
- (var-get (requiring-resolve 'metabase.collections.models.collection/root-collection)))
-
(defn- collection-permission-graph
"Return the permission graph for the collections with id in `collection-ids` and the root collection."
[group-perms]
@@ -155,7 +151,7 @@
collection-id :- [:or [:= :root] ms/PositiveInt]
new-collection-perms :- CollectionPermissions]
(let [collection-id (if (= collection-id :root)
- (assoc (root-collection) :namespace collection-namespace)
+ (assoc collection.root/root-collection :namespace collection-namespace)
collection-id)]
;; remove whatever entry is already there (if any) and add a new entry if applicable
(perms/revoke-collection-permissions! group-id collection-id)
diff --git a/src/metabase/permissions/models/permissions.clj b/src/metabase/permissions/models/permissions.clj
index 09e664d5100..b6bba7da1b6 100644
--- a/src/metabase/permissions/models/permissions.clj
+++ b/src/metabase/permissions/models/permissions.clj
@@ -162,6 +162,7 @@
[clojure.string :as str]
[metabase.api.common :as api]
[metabase.audit-app.core :as audit]
+ [metabase.collections.models.collection.root :as collection.root]
[metabase.config.core :as config]
[metabase.models.interface :as mi]
[metabase.permissions.db :as permissions.db]
@@ -327,7 +328,7 @@
[_model m]
(if-let [collection-id (:collection_id m)]
(mi/can-write? (permissions.db/collection collection-id))
- (mi/can-write? (var-get (requiring-resolve 'metabase.collections.models.collection/root-collection)))))
+ (mi/can-write? collection.root/root-collection)))
;;; +----------------------------------------------------------------------------------------------------------------+
;;; | ENTITY + LIFECYCLE |
diff --git a/src/metabase/permissions/validation.clj b/src/metabase/permissions/validation.clj
index 33746301115..1684ab37ad1 100644
--- a/src/metabase/permissions/validation.clj
+++ b/src/metabase/permissions/validation.clj
@@ -3,6 +3,7 @@
[clojure.string :as str]
[metabase.api.common :as api]
[metabase.config.core :as config]
+ [metabase.permissions.models.permissions :as perms]
[metabase.premium-features.core :as premium-features]
[metabase.util.i18n :refer [tru]]))
@@ -13,10 +14,9 @@
(check-has-application-permission perm-type true))
([perm-type require-superuser?]
- (if-let [f (and (premium-features/enable-advanced-permissions?)
- config/ee-available?
- (requiring-resolve 'metabase-enterprise.advanced-permissions.common/current-user-has-application-permissions?))]
- (api/check-403 (f perm-type))
+ (if (and (premium-features/enable-advanced-permissions?)
+ config/ee-available?)
+ (api/check-403 (perms/current-user-has-application-permissions? perm-type))
(when require-superuser?
(api/check-superuser)))))
diff --git a/src/metabase/premium_features/token_check.clj b/src/metabase/premium_features/token_check.clj
index f4a20616f04..a0cfa5e11aa 100644
--- a/src/metabase/premium_features/token_check.clj
+++ b/src/metabase/premium_features/token_check.clj
@@ -73,7 +73,7 @@
(let [f (fn []
{:post [(integer? %)]}
(log/debug (u/colorize :yellow "GETTING ACTIVE USER COUNT!"))
- (assert ((requiring-resolve 'metabase.app-db.core/db-is-set-up?)) "Metabase DB is not yet set up")
+ (assert (app-db/db-is-set-up?) "Metabase DB is not yet set up")
;; force this to use a new Connection, it seems to be getting called in situations where the Connection
;; is from a different thread and is invalid by the time we get to use it
(let [result (binding [t2.conn/*current-connectable* nil]
@@ -90,7 +90,7 @@
(defn -active-users-count
"Getter for the [[metabase.premium-features.settings/active-users-count]] Setting."
[]
- (if-not ((requiring-resolve 'metabase.app-db.core/db-is-set-up?))
+ (if-not (app-db/db-is-set-up?)
0
(locking-active-user-count)))
@@ -319,7 +319,7 @@
{:pass-thru true})))
;; important to not count these errors against the circuit breaker. These are not the types of errors we need
;; to circuit break. (#65294)
- (when-not ((requiring-resolve 'metabase.app-db.core/db-is-set-up?))
+ (when-not (app-db/db-is-set-up?)
(throw (ex-info "Metabase DB is not yet set up"
{:cause :token-check/app-db-not-ready})))
(locking lock
diff --git a/src/metabase/pulse/events/report_timezone_updated.clj b/src/metabase/pulse/events/report_timezone_updated.clj
index c4f7e75a9e6..ba0b3db27c6 100644
--- a/src/metabase/pulse/events/report_timezone_updated.clj
+++ b/src/metabase/pulse/events/report_timezone_updated.clj
@@ -1,7 +1,7 @@
(ns metabase.pulse.events.report-timezone-updated
(:require
[metabase.events.core :as events]
- [metabase.pulse.task.send-pulses :as send-pulses]
+ [metabase.pulse.task.send-pulses-trigger :as task.send-pulses-trigger]
[methodical.core :as methodical]))
(events/derive! ::event :metabase/event)
@@ -10,4 +10,4 @@
(methodical/defmethod events/publish-event! ::event
"When the report-timezone Setting is updated, update the timezone of all SendPulse triggers."
[_topic _event]
- (send-pulses/update-send-pulse-triggers-timezone!))
+ (task.send-pulses-trigger/update-send-pulse-triggers-timezone!))
diff --git a/src/metabase/pulse/models/pulse_channel.clj b/src/metabase/pulse/models/pulse_channel.clj
index ccc48a1e32e..bc93bbefa27 100644
--- a/src/metabase/pulse/models/pulse_channel.clj
+++ b/src/metabase/pulse/models/pulse_channel.clj
@@ -5,6 +5,7 @@
[metabase.models.interface :as mi]
[metabase.notification.models :as notification.models]
[metabase.pulse.db :as pulse.db]
+ [metabase.pulse.task.send-pulses-trigger :as task.send-pulses-trigger]
[metabase.util :as u]
[metabase.util.i18n :refer [tru]]
[methodical.core :as methodical]
@@ -139,10 +140,6 @@
{:email email})
(get pcid->recipients (:id pc))))))))
-(defn- update-send-pulse-trigger-if-needed!
- [& args]
- (apply (requiring-resolve 'metabase.pulse.task.send-pulses/update-send-pulse-trigger-if-needed!) args))
-
(def ^:dynamic *archive-parent-pulse-when-last-channel-is-deleted*
"Should we automatically archive a Pulse when its last `PulseChannel` is deleted? Normally we do, but this is disabled
in [[update-notification-channels!]] which creates/deletes/updates several channels sequentially."
@@ -158,7 +155,7 @@
(pulse.db/update-pulse! pulse-id {:archived true}))))
;; it's best if this is done in after-delete, but toucan2 doesn't support that yet See toucan2#70S
;; remove this pulse from its existing trigger
- (update-send-pulse-trigger-if-needed! pulse-id pulse-channel :remove-pc-ids #{(:id pulse-channel)}))
+ (task.send-pulses-trigger/update-send-pulse-trigger-if-needed! pulse-id pulse-channel :remove-pc-ids #{(:id pulse-channel)}))
(defn validate-email-domains
"For channels that are being sent to raw email addresses: check that the domains in the emails are allowed by
@@ -204,7 +201,7 @@
[{:keys [pulse_id id] :as pulse-channel}]
(u/prog1 pulse-channel
(when (:enabled pulse-channel)
- (update-send-pulse-trigger-if-needed! pulse_id pulse-channel :add-pc-ids #{id}))))
+ (task.send-pulses-trigger/update-send-pulse-trigger-if-needed! pulse_id pulse-channel :add-pc-ids #{id}))))
(t2/define-before-update :model/PulseChannel
[{:keys [pulse_id id] :as pulse-channel}]
@@ -214,17 +211,17 @@
;; better be done in after-update, but t2/changes isn't available in after-update yet See toucan2#129
(when (some #(contains? #{:schedule_type :schedule_hour :schedule_day :schedule_frame} %) (keys changes))
;; need to remove this PC from the existing trigger
- (update-send-pulse-trigger-if-needed! pulse_id (t2/original pulse-channel)
- :remove-pc-ids #{(:id pulse-channel)})
+ (task.send-pulses-trigger/update-send-pulse-trigger-if-needed! pulse_id (t2/original pulse-channel)
+ :remove-pc-ids #{(:id pulse-channel)})
;; create a new PC with the updated schedule
- (update-send-pulse-trigger-if-needed! pulse_id pulse-channel
- :add-pc-ids #{id}))
+ (task.send-pulses-trigger/update-send-pulse-trigger-if-needed! pulse_id pulse-channel
+ :add-pc-ids #{id}))
(when (contains? changes :enabled)
(if (:enabled changes)
- (update-send-pulse-trigger-if-needed! pulse_id pulse-channel
- :add-pc-ids #{(:id pulse-channel)})
- (update-send-pulse-trigger-if-needed! pulse_id (t2/original pulse-channel)
- :remove-pc-ids #{(:id pulse-channel)}))))
+ (task.send-pulses-trigger/update-send-pulse-trigger-if-needed! pulse_id pulse-channel
+ :add-pc-ids #{(:id pulse-channel)})
+ (task.send-pulses-trigger/update-send-pulse-trigger-if-needed! pulse_id (t2/original pulse-channel)
+ :remove-pc-ids #{(:id pulse-channel)}))))
(validate-email-domains (mi/changes-with-pk pulse-channel)))
;; ## Persistence Functions
diff --git a/src/metabase/pulse/send.clj b/src/metabase/pulse/send.clj
index 50ee7155e3f..791a0dc58f9 100644
--- a/src/metabase/pulse/send.clj
+++ b/src/metabase/pulse/send.clj
@@ -2,6 +2,7 @@
"Code related to sending Pulses (Alerts or Dashboard Subscriptions)."
(:require
[metabase.models.interface :as mi]
+ [metabase.notification.core :as notification]
[metabase.pulse.db :as pulse.db]
[metabase.pulse.models.pulse :as models.pulse]
[metabase.task-history.core :as task-history]
@@ -98,8 +99,6 @@
(update :schedule_frame maybe-name)))}]
:handlers [(get-notification-handler pulse-channel)]}))
-(def ^:private send-notification! (requiring-resolve 'metabase.notification.core/send-notification!))
-
(defn- send-pulse!*
[{:keys [channels channel-ids] :as pulse} dashboard async?]
(let [;; `channel-ids` is the set of channels to send to now, so only send to those. Note the whole set of channels
@@ -108,7 +107,7 @@
channels)]
(doseq [pulse-channel channels]
(try
- (send-notification! (notification-info pulse dashboard pulse-channel) :notification/sync? (not async?))
+ (notification/send-notification! (notification-info pulse dashboard pulse-channel) :notification/sync? (not async?))
(catch Exception e
(log/errorf "[Pulse %d] Error sending to %s channel: %s" (:id pulse) (:channel_type pulse-channel) (ex-message e)))))
nil))
diff --git a/src/metabase/pulse/task/send_pulses.clj b/src/metabase/pulse/task/send_pulses.clj
index 2f7e017ec04..5fb609de8fb 100644
--- a/src/metabase/pulse/task/send_pulses.clj
+++ b/src/metabase/pulse/task/send_pulses.clj
@@ -9,44 +9,22 @@
[clojure.string :as str]
[clojurewerkz.quartzite.conversion :as qc]
[clojurewerkz.quartzite.jobs :as jobs]
- [clojurewerkz.quartzite.schedule.cron :as cron]
[clojurewerkz.quartzite.triggers :as triggers]
- [metabase.driver :as driver]
[metabase.pulse.db :as pulse.db]
[metabase.pulse.models.pulse :as models.pulse]
[metabase.pulse.send :as pulse.send]
- [metabase.query-processor.timezone :as qp.timezone]
+ [metabase.pulse.task.send-pulses-trigger :as task.send-pulses-trigger]
[metabase.task-history.core :as task-history]
[metabase.task.core :as task]
[metabase.tracing.core :as tracing]
- [metabase.util.cron :as u.cron]
- [metabase.util.log :as log]
- [metabase.util.malli :as mu])
+ [metabase.util.log :as log])
(:import
- (java.util TimeZone)
- (org.quartz CronTrigger DisallowConcurrentExecution JobExecutionContext TriggerKey)))
+ (org.quartz DisallowConcurrentExecution JobExecutionContext)))
(set! *warn-on-reflection* true)
;;; ------------------------------------------------ Job: SendPulse ----------------------------------------------------
-(def ^:private send-pulse-job-key (jobs/key "metabase.task.send-pulses.send-pulse.job"))
-
-(defn- send-pulse-trigger-key
- ^TriggerKey [pulse-id schedule-map]
- (triggers/key (format "metabase.task.send-pulse.trigger.%d.%s"
- pulse-id (-> (select-keys schedule-map u.cron/schedule-keys)
- u.cron/schedule-map->cron-string
- (str/replace " " "_")))))
-
-(defn- send-pulse-trigger-key->info
- [trigger-key]
- (let [[_ pulse-id schedule-str] (re-matches #"metabase\.task\.send-pulse\.trigger\.(\d+)\.(.*)" trigger-key)]
- {:pulse-id (parse-long pulse-id)
- :schedule-map (-> schedule-str
- (str/replace "_" " ")
- u.cron/cron-string->schedule-map)}))
-
(defn- send-pulse!
[pulse-id channel-ids]
(tracing/with-span :tasks "task.pulse.send" {:pulse/id pulse-id
@@ -68,41 +46,6 @@
(catch Throwable e
(log/errorf "Error sending Pulse %d to channel ids: %s: %s" pulse-id (str/join ", " channel-ids) (ex-message e))))))
-(defn- send-trigger-timezone
- []
- (or (driver/report-timezone)
- (qp.timezone/system-timezone-id)
- "UTC"))
-
-(mu/defn- send-pulse-trigger
- "Build a Quartz trigger to send a pulse to a list of channel-ids."
- ^CronTrigger
- ([pulse-id :- pos-int?
- schedule-map :- u.cron/ScheduleMap
- pc-ids :- [:set pos-int?]
- timezone :- :string]
- (send-pulse-trigger pulse-id schedule-map pc-ids timezone 6))
- ([pulse-id :- pos-int?
- schedule-map :- u.cron/ScheduleMap
- pc-ids :- [:set pos-int?]
- timezone :- :string
- priority :- pos-int?]
- (triggers/build
- (triggers/with-identity (send-pulse-trigger-key pulse-id schedule-map))
- (triggers/for-job send-pulse-job-key)
- (triggers/using-job-data {"pulse-id" pulse-id
- "channel-ids" pc-ids})
- (triggers/with-schedule
- (cron/schedule
- (cron/cron-schedule (u.cron/schedule-map->cron-string (select-keys schedule-map u.cron/schedule-keys)))
- (cron/in-time-zone (TimeZone/getTimeZone ^String timezone))
- ;; If the trigger is misfired, fire it immediately and proceed with the next scheduled time.
- ;; TODO: upon testing, look like re-firing on startup is not working as expected
- ;;
- ;; See https://www.nurkiewicz.com/2012/04/quartz-scheduler-misfire-instructions.html for more info
- (cron/with-misfire-handling-instruction-fire-and-proceed)))
- (triggers/with-priority priority))))
-
; Clearing pulse channels is not done synchronously in order to support undoing feature.
(defn- clear-pulse-channels-no-recipients!
"Delete PulseChannels that have no recipients and no channel set for a pulse, returns the channel ids that were deleted."
@@ -134,20 +77,6 @@
(send-pulse! pulse-id to-send-enabled-channel-ids)
(log/infof "Skip sending pulse %d because all channels have no recipients" pulse-id))))
-(defn update-send-pulse-triggers-timezone!
- "Update the timezone of all SendPulse triggers if the report timezone changes."
- []
- (let [triggers (-> send-pulse-job-key task/job-info :triggers)
- new-timezone (send-trigger-timezone)]
- (doseq [trigger triggers
- :when (not= new-timezone (:timezone trigger))] ; skip if timezone is the same
- (let [trigger-key (:key trigger)
- channel-ids (get-in trigger [:data "channel-ids"])
- {:keys [pulse-id
- schedule-map]} (send-pulse-trigger-key->info trigger-key)]
- (log/infof "Updating timezone of trigger %s to %s. Was: %s" trigger-key new-timezone (:timezone trigger))
- (task/reschedule-trigger! (send-pulse-trigger pulse-id schedule-map channel-ids new-timezone (:priority trigger)))))))
-
(task/defjob ^{:doc "Triggers that send a pulse to a list of channels at a specific time"}
SendPulse
[context]
@@ -171,73 +100,6 @@
pulse-id trigger-key scheduled-fire-time fire-time recovering? refire-count scheduler-id)
(send-pulse!* pulse-id channel-ids))))
-(declare update-send-pulse-trigger-if-needed!)
-
-(defn- active-dashsub-pcs
- []
- (pulse.db/active-dashboard-subscription-channels))
-
-(defn init-dashboard-subscription-triggers!
- "Update send pulse triggers for all active pulses.
- Called once when Metabase starts up to create triggers for all existing PulseChannels"
- []
- (assert (task/scheduler) "Scheduler must be started before initializing SendPulse triggers")
- (let [trigger-slot->pc-ids (as-> (active-dashsub-pcs) results
- (group-by #(select-keys % [:pulse_id :schedule_type :schedule_day :schedule_hour :schedule_frame]) results)
- (update-vals results #(map :id %)))]
- (doseq [[{:keys [pulse_id] :as schedule-map} pc-ids] trigger-slot->pc-ids]
- (update-send-pulse-trigger-if-needed! pulse_id schedule-map :add-pc-ids (set pc-ids)))))
-
-;;; --------------------------------------------- Helpers -------------------------------------------
-
-;; called by PulseChannel hooks
-(defn update-send-pulse-trigger-if-needed!
- "Update send pulse trigger of a pulse for a specific schedule map with new pulse channel ids.
-
- Send Pulse triggers are grouped by pulse id and schedule time, meaning PulseChannels of a Pulse that scheduled
- to run at the same time of will be send together.
- This function will updates the corresponding trigger if PulseChannels changes.
-
- * To add 2 pulse channels to a trigger
- (update-send-pulse-trigger-if-needed! pulse-id schedule-map :add-pc-ids #{1 2}))
-
- * To remove 2 pulse channels from a trigger
- (update-send-pulse-trigger-if-needed! pulse-id schedule-map :remove-pc-ids #{1 2}))"
- [pulse-id schedule-map & {:keys [add-pc-ids remove-pc-ids]}]
- (let [schedule-map (update-vals (select-keys schedule-map u.cron/schedule-keys)
- #(if (keyword? %)
- (name %)
- %))
- trigger-key (send-pulse-trigger-key pulse-id schedule-map)
- ;; there should be at most one existing trigger
- existing-trigger (->> (-> send-pulse-job-key task/job-info :triggers)
- (filter #(= (:key %) (.getName ^TriggerKey trigger-key)))
- first)
- existing-pc-ids (some-> existing-trigger :data (get "channel-ids") set)
- new-pc-ids (if (some? existing-pc-ids)
- (cond-> existing-pc-ids
- (some? add-pc-ids) (set/union existing-pc-ids (set add-pc-ids))
- (some? remove-pc-ids) (set/difference (set remove-pc-ids)))
- (set add-pc-ids))]
- (cond
- ;; no op when new-pc-ids doesn't change
- (= new-pc-ids existing-pc-ids) nil
-
- ;; delete if no new pc-ids and there is an existing trigger
- (and (empty? new-pc-ids)
- (some? existing-pc-ids))
- (do
- (log/infof "Deleting trigger %s for pulse %d" trigger-key pulse-id)
- (task/delete-trigger! trigger-key))
-
- ;; delete then create if pc ids changes
- (and (seq new-pc-ids)
- (not= new-pc-ids existing-pc-ids))
- (do
- (log/infof "Updating Send Pulse trigger %s for pulse %d with new pc-ids: %s, was: %s " trigger-key pulse-id new-pc-ids existing-pc-ids)
- (task/delete-trigger! trigger-key)
- (task/add-trigger! (send-pulse-trigger pulse-id schedule-map new-pc-ids (send-trigger-timezone)))))))
-
(task/defjob
^{:doc
"Find all notification subscriptions with cron schedules and create a trigger for each.
@@ -252,13 +114,13 @@
InitSendPulseTriggers
[_context]
(log/info "Initializing SendPulse triggers for dashboard subscriptions")
- (init-dashboard-subscription-triggers!))
+ (task.send-pulses-trigger/init-dashboard-subscription-triggers!))
;;; -------------------------------------------------- Task init ------------------------------------------------
(defmethod task/init! ::SendPulses [_]
(let [send-pulse-job (jobs/build
- (jobs/with-identity send-pulse-job-key)
+ (jobs/with-identity task.send-pulses-trigger/send-pulse-job-key)
(jobs/with-description "Send Pulse")
(jobs/of-type SendPulse)
(jobs/store-durably))
diff --git a/src/metabase/pulse/task/send_pulses_trigger.clj b/src/metabase/pulse/task/send_pulses_trigger.clj
new file mode 100644
index 00000000000..f46959a056e
--- /dev/null
+++ b/src/metabase/pulse/task/send_pulses_trigger.clj
@@ -0,0 +1,154 @@
+(ns metabase.pulse.task.send-pulses-trigger
+ "Quartz trigger lifecycle for the SendPulse job.
+
+ Separate from [[metabase.pulse.task.send-pulses]] so that PulseChannel model hooks can (un)schedule triggers: the job
+ body over there loads the sending machinery, which reads the models."
+ (:require
+ [clojure.set :as set]
+ [clojure.string :as str]
+ [clojurewerkz.quartzite.jobs :as jobs]
+ [clojurewerkz.quartzite.schedule.cron :as cron]
+ [clojurewerkz.quartzite.triggers :as triggers]
+ [metabase.driver :as driver]
+ [metabase.pulse.db :as pulse.db]
+ [metabase.query-processor.timezone :as qp.timezone]
+ [metabase.task.core :as task]
+ [metabase.util.cron :as u.cron]
+ [metabase.util.log :as log]
+ [metabase.util.malli :as mu])
+ (:import
+ (java.util TimeZone)
+ (org.quartz CronTrigger TriggerKey)))
+
+(set! *warn-on-reflection* true)
+
+(def send-pulse-job-key
+ "Key of the Quartz job that sends a pulse when one of its triggers fires."
+ (jobs/key "metabase.task.send-pulses.send-pulse.job"))
+
+(defn- send-pulse-trigger-key
+ ^TriggerKey [pulse-id schedule-map]
+ (triggers/key (format "metabase.task.send-pulse.trigger.%d.%s"
+ pulse-id (-> (select-keys schedule-map u.cron/schedule-keys)
+ u.cron/schedule-map->cron-string
+ (str/replace " " "_")))))
+
+(defn- send-pulse-trigger-key->info
+ [trigger-key]
+ (let [[_ pulse-id schedule-str] (re-matches #"metabase\.task\.send-pulse\.trigger\.(\d+)\.(.*)" trigger-key)]
+ {:pulse-id (parse-long pulse-id)
+ :schedule-map (-> schedule-str
+ (str/replace "_" " ")
+ u.cron/cron-string->schedule-map)}))
+
+(defn- send-trigger-timezone
+ []
+ (or (driver/report-timezone)
+ (qp.timezone/system-timezone-id)
+ "UTC"))
+
+(mu/defn- send-pulse-trigger
+ "Build a Quartz trigger to send a pulse to a list of channel-ids."
+ ^CronTrigger
+ ([pulse-id :- pos-int?
+ schedule-map :- u.cron/ScheduleMap
+ pc-ids :- [:set pos-int?]
+ timezone :- :string]
+ (send-pulse-trigger pulse-id schedule-map pc-ids timezone 6))
+ ([pulse-id :- pos-int?
+ schedule-map :- u.cron/ScheduleMap
+ pc-ids :- [:set pos-int?]
+ timezone :- :string
+ priority :- pos-int?]
+ (triggers/build
+ (triggers/with-identity (send-pulse-trigger-key pulse-id schedule-map))
+ (triggers/for-job send-pulse-job-key)
+ (triggers/using-job-data {"pulse-id" pulse-id
+ "channel-ids" pc-ids})
+ (triggers/with-schedule
+ (cron/schedule
+ (cron/cron-schedule (u.cron/schedule-map->cron-string (select-keys schedule-map u.cron/schedule-keys)))
+ (cron/in-time-zone (TimeZone/getTimeZone ^String timezone))
+ ;; If the trigger is misfired, fire it immediately and proceed with the next scheduled time.
+ ;; TODO: upon testing, look like re-firing on startup is not working as expected
+ ;;
+ ;; See https://www.nurkiewicz.com/2012/04/quartz-scheduler-misfire-instructions.html for more info
+ (cron/with-misfire-handling-instruction-fire-and-proceed)))
+ (triggers/with-priority priority))))
+
+(defn update-send-pulse-triggers-timezone!
+ "Update the timezone of all SendPulse triggers if the report timezone changes."
+ []
+ (let [triggers (-> send-pulse-job-key task/job-info :triggers)
+ new-timezone (send-trigger-timezone)]
+ (doseq [trigger triggers
+ :when (not= new-timezone (:timezone trigger))] ; skip if timezone is the same
+ (let [trigger-key (:key trigger)
+ channel-ids (get-in trigger [:data "channel-ids"])
+ {:keys [pulse-id
+ schedule-map]} (send-pulse-trigger-key->info trigger-key)]
+ (log/infof "Updating timezone of trigger %s to %s. Was: %s" trigger-key new-timezone (:timezone trigger))
+ (task/reschedule-trigger! (send-pulse-trigger pulse-id schedule-map channel-ids new-timezone (:priority trigger)))))))
+
+;; called by PulseChannel hooks
+(defn update-send-pulse-trigger-if-needed!
+ "Update send pulse trigger of a pulse for a specific schedule map with new pulse channel ids.
+
+ Send Pulse triggers are grouped by pulse id and schedule time, meaning PulseChannels of a Pulse that scheduled
+ to run at the same time of will be send together.
+ This function will updates the corresponding trigger if PulseChannels changes.
+
+ * To add 2 pulse channels to a trigger
+ (update-send-pulse-trigger-if-needed! pulse-id schedule-map :add-pc-ids #{1 2}))
+
+ * To remove 2 pulse channels from a trigger
+ (update-send-pulse-trigger-if-needed! pulse-id schedule-map :remove-pc-ids #{1 2}))"
+ [pulse-id schedule-map & {:keys [add-pc-ids remove-pc-ids]}]
+ (let [schedule-map (update-vals (select-keys schedule-map u.cron/schedule-keys)
+ #(if (keyword? %)
+ (name %)
+ %))
+ trigger-key (send-pulse-trigger-key pulse-id schedule-map)
+ ;; there should be at most one existing trigger
+ existing-trigger (->> (-> send-pulse-job-key task/job-info :triggers)
+ (filter #(= (:key %) (.getName ^TriggerKey trigger-key)))
+ first)
+ existing-pc-ids (some-> existing-trigger :data (get "channel-ids") set)
+ new-pc-ids (if (some? existing-pc-ids)
+ (cond-> existing-pc-ids
+ (some? add-pc-ids) (set/union existing-pc-ids (set add-pc-ids))
+ (some? remove-pc-ids) (set/difference (set remove-pc-ids)))
+ (set add-pc-ids))]
+ (cond
+ ;; no op when new-pc-ids doesn't change
+ (= new-pc-ids existing-pc-ids) nil
+
+ ;; delete if no new pc-ids and there is an existing trigger
+ (and (empty? new-pc-ids)
+ (some? existing-pc-ids))
+ (do
+ (log/infof "Deleting trigger %s for pulse %d" trigger-key pulse-id)
+ (task/delete-trigger! trigger-key))
+
+ ;; delete then create if pc ids changes
+ (and (seq new-pc-ids)
+ (not= new-pc-ids existing-pc-ids))
+ (do
+ (log/infof "Updating Send Pulse trigger %s for pulse %d with new pc-ids: %s, was: %s " trigger-key pulse-id new-pc-ids existing-pc-ids)
+ (task/delete-trigger! trigger-key)
+ (task/add-trigger! (send-pulse-trigger pulse-id schedule-map new-pc-ids (send-trigger-timezone)))))))
+
+(defn- active-dashsub-pcs
+ []
+ (pulse.db/active-dashboard-subscription-channels))
+
+(defn init-dashboard-subscription-triggers!
+ "Update send pulse triggers for all active pulses.
+ Called once when Metabase starts up to create triggers for all existing PulseChannels"
+ []
+ (assert (task/scheduler) "Scheduler must be started before initializing SendPulse triggers")
+ (let [trigger-slot->pc-ids (as-> (active-dashsub-pcs) results
+ (group-by #(select-keys % [:pulse_id :schedule_type :schedule_day :schedule_hour :schedule_frame]) results)
+ (update-vals results #(map :id %)))]
+ (doseq [[{:keys [pulse_id] :as schedule-map} pc-ids] trigger-slot->pc-ids]
+ (update-send-pulse-trigger-if-needed! pulse_id schedule-map :add-pc-ids (set pc-ids)))))
diff --git a/src/metabase/queries/card.clj b/src/metabase/queries/card.clj
index a09567112e7..547e40a517c 100644
--- a/src/metabase/queries/card.clj
+++ b/src/metabase/queries/card.clj
@@ -8,8 +8,8 @@
[metabase.parameters.field :as parameters.field]
[metabase.parameters.params :as params]
[metabase.parameters.schema :as parameters.schema]
- [metabase.queries.models.card :as card]
[metabase.queries.schema :as queries.schema]
+ [metabase.query-processor.card :as qp.card]
[metabase.util :as u]
[metabase.util.i18n :refer [tru]]
[metabase.util.malli :as mu]
@@ -21,7 +21,7 @@
(u/prog1 (m/find-first #(= (:id %) param-key)
(or (seq (:parameters card))
;; some older cards or cards in e2e just use the template tags on native queries
- (card/template-tag-parameters card)))
+ (qp.card/template-tag-parameters card)))
(when-not <>
(throw (ex-info (tru "Card does not have a parameter with the ID {0}" (pr-str param-key))
{:status-code 400})))))
@@ -46,7 +46,7 @@
slug->param (into {}
(map (juxt (comp keyword :slug) identity))
(or (seq (:parameters card))
- (card/template-tag-parameters card)))]
+ (qp.card/template-tag-parameters card)))]
(vec (for [[slug value] slug->value
;; an empty collection means "no value", as it does in `validate-and-merge-params`, and a parameter
;; with no value constrains nothing whether or not it still exists
diff --git a/src/metabase/queries/core.clj b/src/metabase/queries/core.clj
index 3fcadc1ad7e..42d3b940676 100644
--- a/src/metabase/queries/core.clj
+++ b/src/metabase/queries/core.clj
@@ -10,6 +10,7 @@
[metabase.queries.models.query]
[metabase.queries.models.stored-result]
[metabase.queries.models.stored-result-use]
+ [metabase.query-processor.card]
[potemkin :as p]))
(comment metabase.queries.cached-result/keep-me
@@ -21,7 +22,8 @@
metabase.queries.models.parameter-card/keep-me
metabase.queries.models.query/keep-me
metabase.queries.models.stored-result/keep-me
- metabase.queries.models.stored-result-use/keep-me)
+ metabase.queries.models.stored-result-use/keep-me
+ metabase.query-processor.card/keep-me)
(p/import-vars
[metabase.queries.card
@@ -47,7 +49,6 @@
maybe-unverify!
model-supports-implicit-actions?
model?
- parameter-template-tag?
sole-dashboard-id
starting-card-schema-version
update-card!
@@ -73,7 +74,9 @@
cached-dataset]
[metabase.queries.models.stored-result-use
assert-can-view-card-snapshots!
- carry-pairings-for-document!])
+ carry-pairings-for-document!]
+ [metabase.query-processor.card
+ parameter-template-tag?])
;; the re-exported var carries the docstring; kondo can't see through import-def
#_{:clj-kondo/ignore [:missing-docstring]}
@@ -81,7 +84,7 @@
;; the re-exported var carries the docstring; kondo can't see through import-def
#_{:clj-kondo/ignore [:missing-docstring]}
-(p/import-def metabase.queries.models.card/template-tag-parameters card-template-tag-parameters)
+(p/import-def metabase.query-processor.card/template-tag-parameters card-template-tag-parameters)
;; the re-exported var carries the docstring; kondo can't see through import-def
#_{:clj-kondo/ignore [:missing-docstring]}
diff --git a/src/metabase/queries/models/card.clj b/src/metabase/queries/models/card.clj
index 731e13dd91e..954ceb77746 100644
--- a/src/metabase/queries/models/card.clj
+++ b/src/metabase/queries/models/card.clj
@@ -23,13 +23,11 @@
[metabase.lib.schema :as lib.schema]
[metabase.lib.schema.id :as lib.schema.id]
[metabase.lib.schema.metadata :as lib.schema.metadata]
- [metabase.lib.schema.template-tag :as lib.schema.template-tag]
[metabase.metrics.core :as metrics]
[metabase.models.interface :as mi]
[metabase.models.serialization :as serdes]
[metabase.parameters.core :as parameters]
[metabase.parameters.params :as params]
- [metabase.parameters.schema :as parameters.schema]
[metabase.permissions.core :as perms]
[metabase.premium-features.core :refer [defenterprise]]
[metabase.public-sharing.core :as public-sharing]
@@ -42,6 +40,7 @@
[metabase.search.core :as search]
[metabase.settings.core :as setting]
[metabase.staleness.core :as staleness]
+ [metabase.sync.field-values :as sync.field-values]
[metabase.util :as u]
[metabase.util.embed :refer [maybe-populate-initially-published-at]]
[metabase.util.honey-sql-2 :as h2x]
@@ -335,48 +334,6 @@
(when database-id
{:database_id database-id})))))))))
-;;; TODO -- move this to [[metabase.query-processor.card]] or Lib so the logic can be shared between the backend and
-;;; frontend (?)
-;;;
-;;; NOTE: this should mirror `getTemplateTagParameters` in frontend/src/metabase-lib/parameters/utils/template-tags.ts
-;;; If this function moves you should update the comment that links to this one (#40013)
-;;;
-(mu/defn parameter-template-tag? :- :boolean
- "Whether a parameter is created for this template tag, as opposed to tags that splice content into the query itself,
- like snippets, card references, and tables."
- [{tag-type :type, widget-type :widget-type} :- [:maybe ::lib.schema.template-tag/template-tag]]
- (boolean
- (and tag-type
- (or (contains? lib.schema.template-tag/raw-value-template-tag-types tag-type)
- (= tag-type :temporal-unit)
- (and (= tag-type :dimension) widget-type (not= widget-type :none))))))
-
-;;; TODO -- does this belong HERE or in the `parameters` module?
-(mu/defn template-tag-parameters :- ::parameters.schema/parameters
- "Transforms native query's `template-tags` into `parameters`.
- An older style was to not include `:template-tags` onto cards as parameters. I think this is a mistake and they
- should always be there. Apparently lots of e2e tests are sloppy about this so this is included as a convenience."
- [card :- [:maybe ::queries.schema/card]]
- (for [{tag-type :type, widget-type :widget-type, :as tag} (some-> card :dataset_query not-empty lib/all-template-tags)
- :when (parameter-template-tag? tag)]
- {:id (:id tag)
- :type (or widget-type (case tag-type
- :temporal-unit :temporal-unit
- :date :date/single
- :text :string/=
- :number :number/=
- :boolean :boolean/=
- ;; fallback; should be unreachable since :when filters
- ;; to raw-value-template-tag-types
- :string/=))
- :target (if (contains? #{:dimension :temporal-unit} tag-type)
- [:dimension [:template-tag (:name tag)]]
- [:variable [:template-tag (:name tag)]])
- :name (:display-name tag)
- :slug (:name tag)
- :default (:default tag)
- :required (boolean (:required tag))}))
-
(defn- check-field-filter-fields-are-from-correct-database
"Check that all native query Field filter parameters reference Fields belonging to the Database the query points
against. This is done when saving a Card. The goal here is to prevent people from saving Cards with invalid queries
@@ -614,7 +571,7 @@
"Is Now:" new-param-field-ids
"Newly Added:" newly-added-param-field-ids)
;; Now update the FieldValues for the Fields referenced by this Card.
- ((requiring-resolve 'metabase.sync.field-values/update-field-values-for-on-demand-dbs!) newly-added-param-field-ids)))))
+ (sync.field-values/update-field-values-for-on-demand-dbs! newly-added-param-field-ids)))))
;; updating a model dataset query to not support implicit actions will disable implicit actions if they exist
(when (and (:dataset_query changes)
(= (:type old-card-info) :model)
@@ -821,7 +778,7 @@
(u/prog1 card
(when-let [field-ids (seq (params/card->template-tag-field-ids card))]
(log/info "Card references Fields in params:" field-ids)
- ((requiring-resolve 'metabase.sync.field-values/update-field-values-for-on-demand-dbs!) field-ids))
+ (sync.field-values/update-field-values-for-on-demand-dbs! field-ids))
(parameter-card/upsert-or-delete-from-parameters! "card" (:id card) (:parameters card))))
(defn- apply-dashboard-question-updates [card changes]
diff --git a/src/metabase/query_permissions/impl.clj b/src/metabase/query_permissions/impl.clj
index 46facd4396f..24496b63f35 100644
--- a/src/metabase/query_permissions/impl.clj
+++ b/src/metabase/query_permissions/impl.clj
@@ -20,8 +20,10 @@
[metabase.query-permissions.db :as query-permissions.db]
[metabase.query-processor.error-type :as qp.error-type]
[metabase.query-processor.interface :as qp.i]
+ [metabase.query-processor.preprocess :as qp.preprocess]
;; legacy usage -- don't do things like this going forward
^{:clj-kondo/ignore [:deprecated-namespace :discouraged-namespace]} [metabase.query-processor.store :as qp.store]
+ [metabase.request.core :as request]
[metabase.util :as u]
[metabase.util.i18n :refer [tru]]
[metabase.util.log :as log]
@@ -185,11 +187,8 @@
;; ignore the current user for the purposes of calculating the permissions required to run the query. Don't want the
;; preprocessing to fail because current user doesn't have permissions to run it when we're not trying to run it at
;; all
- (let [do-as-admin (requiring-resolve 'metabase.request.core/do-as-admin)
- preprocess (requiring-resolve 'metabase.query-processor.preprocess/preprocess)]
- (do-as-admin
- (^:once fn* []
- (preprocess query)))))
+ (request/as-admin
+ (qp.preprocess/preprocess query)))
(defn- preprocess-without-per-user-lens
"[[preprocess-query]], minus the preprocess middlewares that resolve the current user's data-access
diff --git a/src/metabase/query_processor/card.clj b/src/metabase/query_processor/card.clj
index ea66b77f9d0..9979e89d700 100644
--- a/src/metabase/query_processor/card.clj
+++ b/src/metabase/query_processor/card.clj
@@ -15,7 +15,7 @@
[metabase.lib.schema.template-tag :as lib.schema.template-tag]
[metabase.parameters.schema :as parameters.schema]
[metabase.premium-features.core :refer [defenterprise]]
- [metabase.queries.core :as queries]
+ [metabase.queries.models.query :as queries.query]
[metabase.queries.schema :as queries.schema]
[metabase.query-processor :as qp]
[metabase.query-processor.error-type :as qp.error-type]
@@ -48,7 +48,7 @@
(defn- enrich-strategy [strategy query]
(case (:type strategy)
- :ttl (let [et (queries/average-execution-time-ms (qp.util/query-hash query))]
+ :ttl (let [et (queries.query/average-execution-time-ms (qp.util/query-hash query))]
(assoc strategy :avg-execution-ms (or et 0)))
strategy))
@@ -140,6 +140,49 @@
here."
false)
+;;; These live here rather than in [[metabase.queries.models.card]] so the QP can use them without
+;;; depending on the Card model.
+(mu/defn parameter-template-tag? :- :boolean
+ "Whether a parameter is created for this template tag, as opposed to tags that splice content into the query itself,
+ like snippets, card references, and tables."
+ [{tag-type :type, widget-type :widget-type} :- [:maybe ::lib.schema.template-tag/template-tag]]
+ (boolean
+ (and tag-type
+ (or (contains? lib.schema.template-tag/raw-value-template-tag-types tag-type)
+ (= tag-type :temporal-unit)
+ (and (= tag-type :dimension) widget-type (not= widget-type :none))))))
+
+;;; TODO -- move this to Lib so the logic can be shared between the backend and frontend (?)
+;;;
+;;; NOTE: this should mirror `getTemplateTagParameters` in frontend/src/metabase-lib/parameters/utils/template-tags.ts
+;;; If this function moves you should update the comment that links to this one (#40013)
+;;;
+;;; TODO -- does this belong HERE or in the `parameters` module?
+(mu/defn template-tag-parameters :- ::parameters.schema/parameters
+ "Transforms native query's `template-tags` into `parameters`.
+ An older style was to not include `:template-tags` onto cards as parameters. I think this is a mistake and they
+ should always be there. Apparently lots of e2e tests are sloppy about this so this is included as a convenience."
+ [card :- [:maybe ::queries.schema/card]]
+ (for [{tag-type :type, widget-type :widget-type, :as tag} (some-> card :dataset_query not-empty lib/all-template-tags)
+ :when (parameter-template-tag? tag)]
+ {:id (:id tag)
+ :type (or widget-type (case tag-type
+ :temporal-unit :temporal-unit
+ :date :date/single
+ :text :string/=
+ :number :number/=
+ :boolean :boolean/=
+ ;; fallback; should be unreachable since :when filters
+ ;; to raw-value-template-tag-types
+ :string/=))
+ :target (if (contains? #{:dimension :temporal-unit} tag-type)
+ [:dimension [:template-tag (:name tag)]]
+ [:variable [:template-tag (:name tag)]])
+ :name (:display-name tag)
+ :slug (:name tag)
+ :default (:default tag)
+ :required (boolean (:required tag))}))
+
(mu/defn- card-template-tag-parameters
"Template tag parameters that have been specified for the Card's `query` (`:dataset_query`), if any, returned as a map
in the format
@@ -265,8 +308,8 @@
is not present in the parameters.
This function ensures that all template-tags are converted to parameters and added to card.parameters."
[{:keys [parameters] :as card} :- ::queries.schema/card]
- (let [template-tag-parameters (queries/card-template-tag-parameters card)
- id->template-tags-parameter (m/index-by :id template-tag-parameters)
+ (let [tag-parameters (template-tag-parameters card)
+ id->template-tags-parameter (m/index-by :id tag-parameters)
parameter-ids (into #{} (map :id) parameters)
;; Preserve the order of card.parameters, merging in template-tag info
merged-parameters (mapv (fn [param]
@@ -278,7 +321,7 @@
;; Append any template-tag parameters not already present in card.parameters
(into merged-parameters
(remove #(contains? parameter-ids (:id %)))
- template-tag-parameters)))
+ tag-parameters)))
(mu/defn- enrich-parameters-from-card :- ::parameters.schema/parameters
"Allow the FE to omit type and target for parameters by adding them from the card."
diff --git a/src/metabase/query_processor/dashboard.clj b/src/metabase/query_processor/dashboard.clj
index 2542d1895cc..e73d87b8721 100644
--- a/src/metabase/query_processor/dashboard.clj
+++ b/src/metabase/query_processor/dashboard.clj
@@ -5,12 +5,12 @@
[clojure.string :as str]
[medley.core :as m]
[metabase.api.common :as api]
- [metabase.dashboards.models.dashboard :as dashboard]
[metabase.dashboards.schema :as dashboards.schema]
[metabase.events.core :as events]
[metabase.lib-metric.core :as lib-metric]
[metabase.lib.core :as lib]
[metabase.lib.schema.id :as lib.schema.id]
+ [metabase.parameters.params :as params]
[metabase.permissions.core :as perms]
[metabase.permissions.metric :as permissions.metric]
[metabase.query-processor.card :as qp.card]
@@ -142,7 +142,7 @@
(defn- dashboard-param-defaults
"Construct parameter entries for any parameters with default values in `dashboard-param-id->param` as returned
- by [[dashboard/dashboard->resolved-params]]."
+ by [[params/dashboard->resolved-params]]."
[dashboard-param-id->param card-id]
(into
{}
@@ -173,7 +173,7 @@
dashcard-id (:id dashcard)
_ (log/tracef "Resolving Dashboard %d Card %d query request parameters" dashboard-id card-id)
request-params (some-> request-params not-empty (->> (lib/normalize ::dashboards.schema/parameters)))
- resolved-params (dashboard/dashboard->resolved-params (assoc dashboard :dashcards [dashcard]))
+ resolved-params (params/dashboard->resolved-params (assoc dashboard :dashcards [dashcard]))
dashboard-param-id->param (into {}
;; remove the `:default` values from Dashboard params. We don't ACTUALLY want to
;; use these values ourselves -- the expectation is that the frontend will pass
diff --git a/src/metabase/query_processor/middleware/annotate.clj b/src/metabase/query_processor/middleware/annotate.clj
index 70859829ee4..c5625ce2ecf 100644
--- a/src/metabase/query_processor/middleware/annotate.clj
+++ b/src/metabase/query_processor/middleware/annotate.clj
@@ -12,7 +12,7 @@
[metabase.lib.schema :as lib.schema]
[metabase.lib.schema.metadata :as lib.schema.metadata]
[metabase.query-processor.debug :as qp.debug]
- [metabase.query-processor.middleware.annotate.legacy-helper-fns]
+ [metabase.query-processor.middleware.annotate.legacy-helper-fns :as annotate.legacy-helper-fns]
[metabase.query-processor.reducible :as qp.reducible]
[metabase.query-processor.schema :as qp.schema]
[metabase.util.malli :as mu]
@@ -20,8 +20,6 @@
[metabase.util.performance :refer [every? mapv empty? get-in]]
[potemkin :as p]))
-(comment metabase.query-processor.middleware.annotate.legacy-helper-fns/keep-me)
-
(mr/def ::col
[:map
[:source {:optional true} ::lib.schema.metadata/column.legacy-source]
@@ -159,6 +157,13 @@
;;; convert drivers to MBQL 5.
#_{:clj-kondo/ignore [:deprecated-var]}
(p/import-vars
- [metabase.query-processor.middleware.annotate.legacy-helper-fns
- aggregation-name
- merged-column-info])
+ [annotate.legacy-helper-fns
+ aggregation-name])
+
+(mu/defn merged-column-info :- ::cols
+ "Returns deduplicated and merged column metadata (`:cols`) for query results by combining (a) the initial results
+ metadata returned by the driver's impl of `execute-reducible-query` and (b) column metadata inferred by logic in
+ this namespace."
+ {:deprecated "0.64.0"}
+ [legacy-query {initial-cols :cols, :as _initial-metadata} :- [:maybe :map]]
+ (expected-cols (annotate.legacy-helper-fns/legacy-query->mbql5-query legacy-query) initial-cols))
diff --git a/src/metabase/query_processor/middleware/annotate/legacy_helper_fns.clj b/src/metabase/query_processor/middleware/annotate/legacy_helper_fns.clj
index 0b0846c7586..ef82686f02d 100644
--- a/src/metabase/query_processor/middleware/annotate/legacy_helper_fns.clj
+++ b/src/metabase/query_processor/middleware/annotate/legacy_helper_fns.clj
@@ -41,6 +41,17 @@
{:inner-query inner-query, :type qp.error-type/qp}
e))))))
+(mu/defn legacy-query->mbql5-query :- ::lib.schema/query
+ "Convert a legacy outer `legacy-query` to an MBQL 5 query. Requires bound QP store."
+ [legacy-query]
+ (lib/query
+ (qp.store/metadata-provider)
+ ;; if this query has a `:native` query added to it already then remove that so we don't get schema validation
+ ;; errors
+ (cond-> legacy-query
+ ((every-pred :native :query) legacy-query)
+ (dissoc :native))))
+
(mu/defn aggregation-name :- ::lib.schema.common/non-blank-string
"Return an appropriate aggregation name/alias *used inside a query* for an `:aggregation` subclause (an aggregation
or expression). Takes an options map as schema won't support passing keypairs directly as a varargs.
@@ -59,19 +70,3 @@
#_{:clj-kondo/ignore [:deprecated-var]}
(legacy-inner-query->mbql5-query legacy-inner-query)
(lib/->mbql5 legacy-ag-clause)))))
-
-(mu/defn merged-column-info :- :metabase.query-processor.middleware.annotate/cols
- "Returns deduplicated and merged column metadata (`:cols`) for query results by combining (a) the initial results
- metadata returned by the driver's impl of `execute-reducible-query` and (b) column metadata inferred by logic in
- this namespace."
- {:deprecated "0.64.0"}
- [legacy-query {initial-cols :cols, :as _initial-metadata} :- [:maybe :map]]
- (let [expected-cols (requiring-resolve 'metabase.query-processor.middleware.annotate/expected-cols)
- mbql5-query (lib/query
- (qp.store/metadata-provider)
- ;; if this query has a `:native` query added to it already then remove that so we don't get
- ;; schema validation errors
- (cond-> legacy-query
- ((every-pred :native :query) legacy-query)
- (dissoc :native)))]
- (expected-cols mbql5-query initial-cols)))
diff --git a/src/metabase/query_processor/middleware/catch_exceptions.clj b/src/metabase/query_processor/middleware/catch_exceptions.clj
index 5a35488c5b0..86b212a3d50 100644
--- a/src/metabase/query_processor/middleware/catch_exceptions.clj
+++ b/src/metabase/query_processor/middleware/catch_exceptions.clj
@@ -6,9 +6,11 @@
[metabase.analytics-interface.core :as analytics]
[metabase.driver :as driver]
[metabase.lib.schema.common :as lib.schema.common]
+ [metabase.query-processor.compile :as qp.compile]
[metabase.query-processor.error-type :as qp.error-type]
[metabase.query-processor.middleware.permissions :as qp.perms]
[metabase.query-processor.pipeline :as qp.pipeline]
+ [metabase.query-processor.preprocess :as qp.preprocess]
[metabase.query-processor.schema :as qp.schema]
[metabase.util :as u]
[metabase.util.i18n :refer [trs]]
@@ -132,9 +134,9 @@
(qp query rff)
(let [extra-info (delay
{:native (u/ignore-exceptions
- ((requiring-resolve 'metabase.query-processor.compile/compile) query))
+ (qp.compile/compile query))
:preprocessed (u/ignore-exceptions
- ((requiring-resolve 'metabase.query-processor.preprocess/preprocess) query))})]
+ (qp.preprocess/preprocess query))})]
(try
(qp query rff)
(catch Throwable e
diff --git a/src/metabase/query_processor/middleware/permissions.clj b/src/metabase/query_processor/middleware/permissions.clj
index 8359d7c543e..89b6412f0a2 100644
--- a/src/metabase/query_processor/middleware/permissions.clj
+++ b/src/metabase/query_processor/middleware/permissions.clj
@@ -9,12 +9,10 @@
[metabase.permissions.core :as perms]
[metabase.premium-features.core :refer [defenterprise]]
[metabase.query-permissions.core :as query-perms]
- [metabase.query-processor.pipeline :as qp.pipeline]
[metabase.query-processor.schema :as qp.schema]
;; the legacy QP pipeline still conveys the metadata provider via the ambient store; no MBQL 5 path yet
^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store]
[metabase.query-processor.util :as qp.util]
- [metabase.util :as u]
[metabase.util.i18n :refer [tru]]
[metabase.util.log :as log]
[metabase.util.malli :as mu]))
@@ -67,20 +65,6 @@
(throw (ex-info (tru "Querying this database requires the audit-app feature flag")
query)))
-(mu/defn remove-internal-keys :- ::lib.schema/query
- "Pre-processing middleware. Strip internal query-processor keys from the incoming `query` so that they can only ever
- be set by the query processor itself, not supplied by a client. Skipped while re-running pivot sub-queries, which
- legitimately carry these keys."
- [query :- ::lib.schema/query]
- (cond-> query
- (not qp.pipeline/*pivot?*) lib/prepare-after-deserialization))
-
-(mu/defn record-referenced-card-ids :- ::lib.schema/query
- "Pre-processing middleware. Record the source-card IDs referenced by `query` under the
- `:query-permissions/referenced-card-ids` key."
- [query :- ::lib.schema/query]
- (u/assoc-dissoc query :query-permissions/referenced-card-ids (lib/all-source-card-ids-recursive query)))
-
(mu/defn check-query-permissions*
"Check that User with `user-id` has permissions to run `query`, or throw an exception."
[query :- ::qp.schema/any-query]
diff --git a/src/metabase/query_processor/middleware/permissions/preprocess.clj b/src/metabase/query_processor/middleware/permissions/preprocess.clj
new file mode 100644
index 00000000000..67c7cd2c9c7
--- /dev/null
+++ b/src/metabase/query_processor/middleware/permissions/preprocess.clj
@@ -0,0 +1,28 @@
+(ns metabase.query-processor.middleware.permissions.preprocess
+ "Pre-processing middleware that prepares an incoming query for the permission checks that run later.
+
+ Separate from [[metabase.query-processor.middleware.permissions]] because that namespace needs the
+ `query-permissions` module to do the checking, and `query-permissions` in turn
+ needs [[metabase.query-processor.preprocess]] to resolve the sources of a query."
+ (:require
+ [metabase.lib.core :as lib]
+ [metabase.lib.schema :as lib.schema]
+ [metabase.query-processor.pipeline :as qp.pipeline]
+ [metabase.util :as u]
+ [metabase.util.malli :as mu]))
+
+(set! *warn-on-reflection* true)
+
+(mu/defn remove-internal-keys :- ::lib.schema/query
+ "Pre-processing middleware. Strip internal query-processor keys from the incoming `query` so that they can only ever
+ be set by the query processor itself, not supplied by a client. Skipped while re-running pivot sub-queries, which
+ legitimately carry these keys."
+ [query :- ::lib.schema/query]
+ (cond-> query
+ (not qp.pipeline/*pivot?*) lib/prepare-after-deserialization))
+
+(mu/defn record-referenced-card-ids :- ::lib.schema/query
+ "Pre-processing middleware. Record the source-card IDs referenced by `query` under the
+ `:query-permissions/referenced-card-ids` key."
+ [query :- ::lib.schema/query]
+ (u/assoc-dissoc query :query-permissions/referenced-card-ids (lib/all-source-card-ids-recursive query)))
diff --git a/src/metabase/query_processor/preprocess.clj b/src/metabase/query_processor/preprocess.clj
index eaff3e0b7f7..fc91aae79eb 100644
--- a/src/metabase/query_processor/preprocess.clj
+++ b/src/metabase/query_processor/preprocess.clj
@@ -30,7 +30,7 @@
[metabase.query-processor.middleware.normalize-query :as normalize]
[metabase.query-processor.middleware.optimize-temporal-filters :as optimize-temporal-clauses]
[metabase.query-processor.middleware.parameters :as parameters]
- [metabase.query-processor.middleware.permissions :as qp.perms]
+ [metabase.query-processor.middleware.permissions.preprocess :as qp.perms.preprocess]
[metabase.query-processor.middleware.persistence :as qp.persistence]
[metabase.query-processor.middleware.prefetch-metadata :as prefetch-metadata]
[metabase.query-processor.middleware.reconcile-breakout-and-order-by-bucketing :as reconcile-bucketing]
@@ -60,8 +60,8 @@
All of these middlewares assume MBQL 5."
;; ↓↓↓ PRE-PROCESSING ↓↓↓ happens from TOP TO BOTTOM
[#'normalize/normalize-preprocessing-middleware
- #'qp.perms/remove-internal-keys
- #'qp.perms/record-referenced-card-ids
+ #'qp.perms.preprocess/remove-internal-keys
+ #'qp.perms.preprocess/record-referenced-card-ids
#'qp.constraints/maybe-add-default-userland-constraints
#'validate/validate-query
#'prefetch-metadata/prefetch-metadata
diff --git a/src/metabase/query_processor/util/persisted_cache.clj b/src/metabase/query_processor/util/persisted_cache.clj
index 490b3f8524b..8e47f84070d 100644
--- a/src/metabase/query_processor/util/persisted_cache.clj
+++ b/src/metabase/query_processor/util/persisted_cache.clj
@@ -4,6 +4,8 @@
(:require
[metabase.driver :as driver]
[metabase.driver.ddl.interface :as ddl.i]
+ ;; We should not be using specific driver implementations
+ [metabase.driver.sql.util :as sql.u]
[metabase.driver.util :as driver.u]
[metabase.lib.schema.id :as lib.schema.id]
[metabase.lib.schema.metadata :as lib.schema.metadata]
@@ -32,16 +34,14 @@
persistence is appropriate. Use [[can-substitute?]] for that check."
[database-id :- ::lib.schema.id/database
{:keys [table-name] :as _persisted-info} :- ::lib.schema.metadata/persisted-info]
- (let [driver (or driver/*driver* (driver.u/database->driver database-id))
- ;; We should not be using specific driver implementations
- quote-name (requiring-resolve 'metabase.driver.sql.util/quote-name)]
+ (let [driver (or driver/*driver* (driver.u/database->driver database-id))]
;; select * because we don't actually know the name of the fields when in the actual query. See #28902
(format "select * from %s.%s"
- (quote-name
+ (sql.u/quote-name
driver
:table
(ddl.i/schema-name {:id database-id} (system/site-uuid)))
- (quote-name
+ (sql.u/quote-name
driver
:table
table-name))))
diff --git a/src/metabase/search/impl.clj b/src/metabase/search/impl.clj
index 455149a23dd..331eb091d5d 100644
--- a/src/metabase/search/impl.clj
+++ b/src/metabase/search/impl.clj
@@ -1,6 +1,7 @@
(ns metabase.search.impl
(:require
[clojure.string :as str]
+ [metabase.api.common :as api]
[metabase.collections.models.collection :as collection]
[metabase.collections.models.collection.root :as collection.root]
[metabase.models.interface :as mi]
@@ -43,8 +44,8 @@
on dynamic variables."
{:style/indent 0}
[current-user-id current-user-perms & body]
- `(with-bindings {(requiring-resolve 'metabase.api.common/*current-user-id*) ~current-user-id
- (requiring-resolve 'metabase.api.common/*current-user-permissions-set*) (atom ~current-user-perms)}
+ `(binding [api/*current-user-id* ~current-user-id
+ api/*current-user-permissions-set* (atom ~current-user-perms)]
~@body))
(defn- can-write? [{:keys [current-user-id current-user-perms]} instance]
diff --git a/src/metabase/search/in_place/filter.clj b/src/metabase/search/in_place/filter.clj
index 0365e68946a..211b30a0fa7 100644
--- a/src/metabase/search/in_place/filter.clj
+++ b/src/metabase/search/in_place/filter.clj
@@ -19,6 +19,7 @@
[metabase.query-processor.parameters.dates :as params.dates]
[metabase.search.config :as search.config :refer [SearchableModel SearchContext]]
[metabase.search.filter :as search.filter]
+ [metabase.search.in-place.search-model :as search-model]
[metabase.search.in-place.util :as search.util]
[metabase.search.permissions :as search.permissions]
[metabase.search.spec :as search.spec]
@@ -74,8 +75,7 @@
(when-let [query (:search-string search-context)]
(into
[:or]
- (for [column (->> (let [search-columns-fn (requiring-resolve 'metabase.search.in-place.legacy/searchable-columns)]
- (search-columns-fn model search-native-query))
+ (for [column (->> (search-model/searchable-columns model search-native-query)
(map #(search.config/column-with-model-alias model %)))
wildcarded-token (->> (search.util/normalize query)
search.util/tokenize
@@ -267,9 +267,6 @@
(->> (get query join-type) (partition 2) (map first) (some #(= % table)) boolean))
;; We won't need this post-legacy as it defines the joins à la carte.
-(defn- search-model->revision-model [model]
- ((requiring-resolve 'metabase.search.in-place.legacy/search-model->revision-model) model))
-
(doseq [model ["dashboard" "card" "dataset" "metric"]]
(defmethod build-optional-filter-query [:last-edited-by model]
[_filter model query editor-ids]
@@ -278,7 +275,7 @@
(not (joined-with-table? query :join :revision))
(-> (sql.helpers/join :revision [:= :revision.model_id (search.config/column-with-model-alias model :id)])
(sql.helpers/where [:= :revision.most_recent true]
- [:= :revision.model (search-model->revision-model model)]))
+ [:= :revision.model (search-model/search-model->revision-model model)]))
(= 1 (count editor-ids))
(sql.helpers/where [:= :revision.user_id (first editor-ids)])
@@ -293,7 +290,7 @@
(not (joined-with-table? query :join :revision))
(-> (sql.helpers/join :revision [:= :revision.model_id (search.config/column-with-model-alias model :id)])
(sql.helpers/where [:= :revision.most_recent true]
- [:= :revision.model (search-model->revision-model model)]))
+ [:= :revision.model (search-model/search-model->revision-model model)]))
true
;; on UI we showed the the last edit info from revision.timestamp
;; not the model.updated_at column
@@ -362,7 +359,7 @@
(-> (merge
;; models support search-native-query if there are additional columns to search when the `search-native-query`
;; argument is true
- {:search-native-query (->> (dissoc (methods @(requiring-resolve 'metabase.search.in-place.legacy/searchable-columns)) :default)
+ {:search-native-query (->> (dissoc (methods search-model/searchable-columns) :default)
(filter (fn [[model f]]
(seq (set/difference (set (f model true)) (set (f model false))))))
(map first)
diff --git a/src/metabase/search/in_place/legacy.clj b/src/metabase/search/in_place/legacy.clj
index 817e2749e80..b31b002f77b 100644
--- a/src/metabase/search/in_place/legacy.clj
+++ b/src/metabase/search/in_place/legacy.clj
@@ -14,6 +14,7 @@
[metabase.search.filter :as search.filter]
[metabase.search.in-place.filter :as search.in-place.filter]
[metabase.search.in-place.scoring :as scoring]
+ [metabase.search.in-place.search-model :as search-model]
[metabase.search.in-place.util :as search.util]
[metabase.search.permissions :as search.permissions]
[metabase.util.honey-sql-2 :as h2x]
@@ -35,14 +36,6 @@
(when (seq terms)
[(str/join " " terms)]))
-(defn search-model->revision-model
- "Return the appropriate revision model given a search model."
- [model]
- (case model
- "dataset" (recur "card")
- "metric" (recur "card")
- (str/capitalize model)))
-
(mu/defn- ->column-alias :- keyword?
"Returns the column name. If the column is aliased, i.e. [`:original_name` `:aliased_name`], return the aliased
column name"
@@ -247,7 +240,7 @@
(sql.helpers/left-join [:revision :r]
[:and [:= :r.model_id (search.config/column-with-model-alias model :id)]
[:= :r.most_recent true]
- [:= :r.model (search-model->revision-model model)]])))
+ [:= :r.model (search-model/search-model->revision-model model)]])))
(mu/defn- with-moderated-status :- :map
[query :- :map
@@ -285,85 +278,6 @@
{:arglists '([model search-context])}
(fn [model _] model))
-(defmulti searchable-columns
- "The columns that can be searched for each model."
- {:arglists '([model search-native-query])}
- (fn [model _] model))
-
-(defmethod searchable-columns :default
- [_ _]
- [:name])
-
-(defmethod searchable-columns "action"
- [_ search-native-query]
- (cond-> [:name
- :description]
- search-native-query
- (conj :dataset_query)))
-
-(defmethod searchable-columns "card"
- [_ search-native-query]
- (cond-> [:name
- :description]
- search-native-query
- (conj :dataset_query)))
-
-(defmethod searchable-columns "dataset"
- [_ search-native-query]
- (searchable-columns "card" search-native-query))
-
-(defmethod searchable-columns "measure"
- [_ _]
- [:name
- :description])
-
-(defmethod searchable-columns "metric"
- [_ search-native-query]
- (searchable-columns "card" search-native-query))
-
-(defmethod searchable-columns "dashboard"
- [_ _]
- [:name
- :description])
-
-(defmethod searchable-columns "page"
- [_ search-native-query]
- (searchable-columns "dashboard" search-native-query))
-
-(defmethod searchable-columns "database"
- [_ _]
- [:name
- :description])
-
-(defmethod searchable-columns "table"
- [_ _]
- [:name
- :display_name
- :description])
-
-(defmethod searchable-columns "transform"
- [_ search-native-query]
- (cond-> [:name
- :description]
- search-native-query
- (conj :source)))
-
-(defmethod searchable-columns "indexed-entity"
- [_ _]
- [:name])
-
-(defmethod searchable-columns "document"
- [_ _]
- [:name
- :document])
-
-;; mirrors the appdb spec's :search-terms [:name :description] (see
-;; metabase.explorations.models.exploration)
-(defmethod searchable-columns "exploration"
- [_ _]
- [:name
- :description])
-
(def ^:private default-columns
"Columns returned for all models."
[:id :name :description :archived :created_at :updated_at])
diff --git a/src/metabase/search/in_place/scoring.clj b/src/metabase/search/in_place/scoring.clj
index da965c3239b..0d55b12e562 100644
--- a/src/metabase/search/in_place/scoring.clj
+++ b/src/metabase/search/in_place/scoring.clj
@@ -115,6 +115,7 @@
[java-time.api :as t]
[metabase.premium-features.core :refer [defenterprise]]
[metabase.search.config :as search.config]
+ [metabase.search.in-place.search-model :as search-model]
[metabase.search.in-place.util :as search.util]
[metabase.util :as u]))
@@ -153,19 +154,12 @@
{:is_match is-match
:text (tokens->string text-tokens (not is-match))})))))
-(let [resolved-fn (delay (requiring-resolve 'metabase.search.in-place.legacy/searchable-columns))]
- (defn- searchable-columns-fn
- "Get searchable columns for a model.
- Lazily resolves the function from the legacy namespace to avoid circular dependencies."
- [model search-native-query]
- (@resolved-fn model search-native-query)))
-
(defn- text-scores-with
"Scores a search result. Returns a vector of score maps, each containing `:weight`, `:score`, and other info about
the text match, if there is one. If there is no match, the score is 0."
[search-native-query weighted-scorers query-tokens search-result]
;; TODO is pmap over search-result worth it?
- (let [scores (for [column (searchable-columns-fn (:model search-result) search-native-query)
+ (let [scores (for [column (search-model/searchable-columns (:model search-result) search-native-query)
{:keys [scorer name weight]
:as _ws} weighted-scorers
:let [matched-text (-> search-result
diff --git a/src/metabase/search/in_place/search_model.clj b/src/metabase/search/in_place/search_model.clj
new file mode 100644
index 00000000000..4acb8619411
--- /dev/null
+++ b/src/metabase/search/in_place/search_model.clj
@@ -0,0 +1,95 @@
+(ns metabase.search.in-place.search-model
+ "Per-search-model facts shared by the in-place engine's namespaces.
+
+ These live here rather than in [[metabase.search.in-place.legacy]] because
+ [[metabase.search.in-place.filter]] and [[metabase.search.in-place.scoring]] both need them, and `legacy` reads
+ those two namespaces."
+ (:require
+ [clojure.string :as str]))
+
+(defn search-model->revision-model
+ "Return the appropriate revision model given a search model."
+ [model]
+ (case model
+ "dataset" (recur "card")
+ "metric" (recur "card")
+ (str/capitalize model)))
+
+(defmulti searchable-columns
+ "The columns that can be searched for each model."
+ {:arglists '([model search-native-query])}
+ (fn [model _] model))
+
+(defmethod searchable-columns :default
+ [_ _]
+ [:name])
+
+(defmethod searchable-columns "action"
+ [_ search-native-query]
+ (cond-> [:name
+ :description]
+ search-native-query
+ (conj :dataset_query)))
+
+(defmethod searchable-columns "card"
+ [_ search-native-query]
+ (cond-> [:name
+ :description]
+ search-native-query
+ (conj :dataset_query)))
+
+(defmethod searchable-columns "dataset"
+ [_ search-native-query]
+ (searchable-columns "card" search-native-query))
+
+(defmethod searchable-columns "measure"
+ [_ _]
+ [:name
+ :description])
+
+(defmethod searchable-columns "metric"
+ [_ search-native-query]
+ (searchable-columns "card" search-native-query))
+
+(defmethod searchable-columns "dashboard"
+ [_ _]
+ [:name
+ :description])
+
+(defmethod searchable-columns "page"
+ [_ search-native-query]
+ (searchable-columns "dashboard" search-native-query))
+
+(defmethod searchable-columns "database"
+ [_ _]
+ [:name
+ :description])
+
+(defmethod searchable-columns "table"
+ [_ _]
+ [:name
+ :display_name
+ :description])
+
+(defmethod searchable-columns "transform"
+ [_ search-native-query]
+ (cond-> [:name
+ :description]
+ search-native-query
+ (conj :source)))
+
+(defmethod searchable-columns "indexed-entity"
+ [_ _]
+ [:name])
+
+(defmethod searchable-columns "document"
+ [_ _]
+ [:name
+ :document])
+
+;; mirrors the appdb spec's :search-terms [:name :description] (see
+;; metabase.explorations.models.exploration)
+(defmethod searchable-columns "exploration"
+ [_ _]
+ [:name
+ :description])
diff --git a/src/metabase/search/permissions.clj b/src/metabase/search/permissions.clj
index 122960d0899..5429f92d437 100644
--- a/src/metabase/search/permissions.clj
+++ b/src/metabase/search/permissions.clj
@@ -1,5 +1,6 @@
(ns metabase.search.permissions
(:require
+ [metabase.api.common :as api]
[metabase.collections.models.collection :as collection]
[metabase.models.interface :as mi]
[metabase.permissions.core :as perms]
@@ -7,7 +8,7 @@
[metabase.util.malli :as mu]))
(defn- assert-current-user! [missing-param]
- (assert @@(requiring-resolve 'metabase.api.common/*current-user*)
+ (assert @api/*current-user*
(format "metabase.api.common/*current-user* must be bound if %s is missing from search-ctx" missing-param)))
(defn- impersonated-user? [{:keys [is-impersonated-user?] :as _search-ctx}]
diff --git a/src/metabase/settings/models/setting.clj b/src/metabase/settings/models/setting.clj
index 6a3b6cd95f1..dfa4c81e512 100644
--- a/src/metabase/settings/models/setting.clj
+++ b/src/metabase/settings/models/setting.clj
@@ -10,6 +10,7 @@
[malli.core :as mc]
[medley.core :as m]
[metabase.api.common :as api]
+ [metabase.app-db.core :as app-db]
[metabase.app-db.setting :as mdb.setting]
[metabase.config.core :as config]
[metabase.events.core :as events]
@@ -535,14 +536,6 @@
(binding [*disable-init* true]
(get setting-definition-or-name)))
-(def ^:private db-is-set-up-var (atom nil))
-
-(defn- db-is-set-up? []
- ;; this should never be hit. it is just overly cautious against a NPE here. But no way this cannot resolve
- (let [f (or @db-is-set-up-var
- (reset! db-is-set-up-var (requiring-resolve 'metabase.app-db.core/db-is-set-up?)))]
- (if f (f) false)))
-
(defn- db-or-cache-value*
"Look up a single setting key in the DB or cache. Returns the raw (possibly empty) string, or nil."
^String [setting-name-str]
@@ -567,7 +560,7 @@
^String [setting-definition-or-name]
(let [setting (resolve-setting setting-definition-or-name)]
;; cannot use db (and cache populated from db) if db is not set up
- (when (and (db-is-set-up?) (allows-site-wide-values? setting))
+ (when (and (app-db/db-is-set-up?) (allows-site-wide-values? setting))
(or (not-empty (db-or-cache-value* (setting-name setting)))
(when-let [deprecated-name (:deprecated-name setting)]
(when-let [v (not-empty (db-or-cache-value* (setting-name deprecated-name)))]
@@ -591,7 +584,7 @@
(defn- init! [setting-definition-or-name]
(let [{:keys [init] :as setting} (resolve-setting setting-definition-or-name)]
(when init
- (when (not (db-is-set-up?))
+ (when (not (app-db/db-is-set-up?))
(throw (ex-info "Cannot initialize setting before the db is set up" {:setting setting})))
;; We do not need to interact with the restore-cache-lock as it is OK to race with it.
(if-not (.tryLock init-lock 30 TimeUnit/SECONDS)
diff --git a/src/metabase/sso/core.clj b/src/metabase/sso/core.clj
index cb82ef2df72..992f885ba24 100644
--- a/src/metabase/sso/core.clj
+++ b/src/metabase/sso/core.clj
@@ -4,6 +4,7 @@
[metabase.sso.google]
[metabase.sso.ldap]
[metabase.sso.ldap.default-implementation]
+ [metabase.sso.ldap.settings]
[metabase.sso.oidc.check]
[metabase.sso.oidc.state]
[metabase.sso.settings]
@@ -26,10 +27,11 @@
[metabase.sso.settings
google-auth-client-id
google-auth-enabled
- ldap-enabled
send-new-sso-user-admin-email?
sso-enabled?
- sso-source-enabled?])
+ sso-source-enabled?]
+ [metabase.sso.ldap.settings
+ ldap-enabled])
;; the re-exported var carries the docstring; kondo can't see through import-def
#_{:clj-kondo/ignore [:missing-docstring]}
diff --git a/src/metabase/sso/init.clj b/src/metabase/sso/init.clj
index a0718f33e4c..fee8efa4235 100644
--- a/src/metabase/sso/init.clj
+++ b/src/metabase/sso/init.clj
@@ -1,5 +1,6 @@
(ns metabase.sso.init
(:require
+ [metabase.sso.ldap.settings]
[metabase.sso.providers.google]
[metabase.sso.providers.ldap]
[metabase.sso.providers.oidc]
diff --git a/src/metabase/sso/ldap/settings.clj b/src/metabase/sso/ldap/settings.clj
new file mode 100644
index 00000000000..89196913dcb
--- /dev/null
+++ b/src/metabase/sso/ldap/settings.clj
@@ -0,0 +1,29 @@
+(ns metabase.sso.ldap.settings
+ "The `ldap-enabled` setting, which refuses to turn LDAP on until the configured server actually answers.
+
+ It lives here rather than alongside the other SSO settings because that check runs
+ [[metabase.sso.ldap/test-current-ldap-details]], while [[metabase.sso.ldap]] reads the LDAP connection settings
+ from [[metabase.sso.settings]] -- defining it there would make those two namespaces require each other.
+
+ Nothing refers to this namespace by name; the setting reaches the settings API only because
+ [[metabase.sso.init]] loads it."
+ (:require
+ [metabase.settings.core :as setting :refer [defsetting]]
+ [metabase.sso.ldap :as ldap]
+ [metabase.util.i18n :refer [deferred-tru tru]]))
+
+(defsetting ldap-enabled
+ (deferred-tru "Is LDAP currently enabled?")
+ :type :boolean
+ :visibility :public
+ :setter (fn [new-value]
+ (let [new-value (boolean new-value)]
+ (when new-value
+ ;; Test the LDAP settings before enabling
+ (let [result (ldap/test-current-ldap-details)]
+ (when-not (= :SUCCESS (:status result))
+ (throw (ex-info (tru "Unable to connect to LDAP server with current settings")
+ (ldap/humanize-error-messages result))))))
+ (setting/set-value-of-type! :boolean :ldap-enabled new-value)))
+ :default false
+ :audit :getter)
diff --git a/src/metabase/sso/providers/slack_connect.clj b/src/metabase/sso/providers/slack_connect.clj
index 937cdf73a00..e9bb3ed563b 100644
--- a/src/metabase/sso/providers/slack_connect.clj
+++ b/src/metabase/sso/providers/slack_connect.clj
@@ -6,6 +6,7 @@
[metabase.server.settings :as server.settings]
[metabase.sso.db :as sso.db]
[metabase.sso.settings :as sso-settings]
+ [metabase.system.core :as system]
[metabase.util.i18n :refer [tru]]
[methodical.core :as methodical]))
@@ -82,7 +83,7 @@
[redirect-url]
(try
(let [redirect (some-> redirect-url (java.net.URI.))
- our-host (some-> ((requiring-resolve 'metabase.system.core/site-url)) (java.net.URI.) (.getHost))]
+ our-host (some-> (system/site-url) (java.net.URI.) (.getHost))]
(when-not (or (nil? redirect-url)
(and (nil? (.getHost redirect))
(nil? (.getScheme redirect)))
diff --git a/src/metabase/sso/settings.clj b/src/metabase/sso/settings.clj
index 8e7f5c6a7a8..216462a2415 100644
--- a/src/metabase/sso/settings.clj
+++ b/src/metabase/sso/settings.clj
@@ -128,22 +128,6 @@
(ldap-user-base))))
:doc false)
-(defsetting ldap-enabled
- (deferred-tru "Is LDAP currently enabled?")
- :type :boolean
- :visibility :public
- :setter (fn [new-value]
- (let [new-value (boolean new-value)]
- (when new-value
- ;; Test the LDAP settings before enabling
- (let [result ((requiring-resolve 'metabase.sso.ldap/test-current-ldap-details))]
- (when-not (= :SUCCESS (:status result))
- (throw (ex-info (tru "Unable to connect to LDAP server with current settings")
- ((requiring-resolve 'metabase.sso.ldap/humanize-error-messages) result))))))
- (setting/set-value-of-type! :boolean :ldap-enabled new-value)))
- :default false
- :audit :getter)
-
(defsetting ldap-timeout-seconds
(deferred-tru "Maximum time, in seconds, to wait for LDAP server before falling back to local authentication")
:type :double
@@ -296,7 +280,9 @@
"Any SSO provider is configured and enabled"
[]
(or (google-auth-enabled)
- (ldap-enabled)
+ ;; read by keyword the way `ee-sso-configured?` above does: `ldap-enabled` is defined in
+ ;; [[metabase.sso.ldap.settings]], which this namespace cannot require
+ (setting/get :ldap-enabled)
(ee-sso-configured?)))
(defn sso-source-enabled?
@@ -307,7 +293,7 @@
(boolean
(case (keyword sso-source)
:google (google-auth-enabled)
- :ldap (ldap-enabled)
+ :ldap (setting/get :ldap-enabled)
;; Enterprise SSO providers: setting/get respects the :feature flag on each
;; setting — returning the default (false) when the feature is unlicensed (e.g.,
;; after license downgrade), so users aren't locked out of password reset.
diff --git a/src/metabase/sync/fetch_metadata.clj b/src/metabase/sync/fetch_metadata.clj
index 605e9c960f0..afa7c6e7194 100644
--- a/src/metabase/sync/fetch_metadata.clj
+++ b/src/metabase/sync/fetch_metadata.clj
@@ -5,6 +5,7 @@
(:require
[clojure.set :as set]
[metabase.driver :as driver]
+ [metabase.driver.sql-jdbc.sync :as sql-jdbc.sync]
[metabase.driver.util :as driver.u]
[metabase.lib-be.core :as lib-be]
[metabase.sync.db :as sync.db]
@@ -37,7 +38,7 @@
(let [driver (driver.u/database->driver database)]
(cond-> fields
(driver.u/supports? driver :nested-field-columns database)
- (set/union ((requiring-resolve 'metabase.driver.sql-jdbc.sync/describe-nested-field-columns) driver database table)))))
+ (set/union (sql-jdbc.sync/describe-nested-field-columns driver database table)))))
(mu/defn table-fields-metadata :- [:set i/TableMetadataField]
"Fetch metadata about Fields belonging to a given `table` directly from an external database by calling its driver's
diff --git a/src/metabase/sync/task/sync_databases.clj b/src/metabase/sync/task/sync_databases.clj
index 37d70e46be5..86b95ec252b 100644
--- a/src/metabase/sync/task/sync_databases.clj
+++ b/src/metabase/sync/task/sync_databases.clj
@@ -6,8 +6,6 @@
(:require
[clojurewerkz.quartzite.conversion :as qc]
[clojurewerkz.quartzite.jobs :as jobs]
- [clojurewerkz.quartzite.schedule.cron :as cron]
- [clojurewerkz.quartzite.triggers :as triggers]
[java-time.api :as t]
[metabase.audit-app.core :as audit]
[metabase.config.core :as config]
@@ -21,10 +19,10 @@
[metabase.sync.field-values :as sync.field-values]
[metabase.sync.schedules :as sync.schedules]
[metabase.sync.sync-metadata :as sync-metadata]
+ [metabase.sync.task.sync-databases-trigger :as sync-databases-trigger]
[metabase.task.core :as task]
[metabase.tracing.core :as tracing]
[metabase.util :as u]
- [metabase.util.cron :as u.cron]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
[metabase.util.malli.registry :as mr]
@@ -32,10 +30,7 @@
[metabase.warehouses.core :as warehouses])
(:import
(org.quartz
- CronTrigger
- JobDetail
- JobKey
- TriggerKey)))
+ JobDetail)))
(set! *warn-on-reflection* true)
@@ -43,8 +38,6 @@
;;; | JOB LOGIC |
;;; +----------------------------------------------------------------------------------------------------------------+
-(declare unschedule-tasks-for-db!)
-
(mu/defn- job-context->database-id :- [:maybe ::lib.schema.id/database]
"Get the Database ID referred to in `job-context`."
[job-context]
@@ -78,7 +71,7 @@
(log/infof "Starting sync task for Database %d." database-id)
(when-let [database (or (sync.db/database database-id)
(do
- (unschedule-tasks-for-db! (mi/instance :model/Database {:id database-id}))
+ (sync-databases-trigger/unschedule-tasks-for-db! (mi/instance :model/Database {:id database-id}))
(log/warnf "Cannot sync Database %d: Database does not exist." database-id)))]
(if-let [ex (try
;; it's okay to allow testing H2 connections during sync. We only want to disallow you from testing them for the
@@ -143,7 +136,7 @@
(log/infof "Update Field values task triggered for Database %d." database-id)
(when-let [database (or (sync.db/database database-id)
(do
- (unschedule-tasks-for-db! (mi/instance :model/Database {:id database-id}))
+ (sync-databases-trigger/unschedule-tasks-for-db! (mi/instance :model/Database {:id database-id}))
(log/warnf "Cannot update Field values for Database %d: Database does not exist." database-id)))]
(if (:is_full_sync database)
(sync.field-values/update-field-values! database)
@@ -155,188 +148,25 @@
(update-field-values! job-context))
;;; +----------------------------------------------------------------------------------------------------------------+
-;;; | TASK INFO AND GETTER FUNCTIONS |
+;;; | JOB DEFINITIONS |
;;; +----------------------------------------------------------------------------------------------------------------+
(mr/def ::class
[:fn {:error/message "a Class"} class?])
-(def ^:private TaskInfo
- "One-off schema for information about the various sync tasks we run for a DB."
- [:map
- [:key :keyword]
- [:db-schedule-column :keyword]
- [:job-class ::class]
- [:name :string]])
-
-(def ^:private sync-analyze-task-info
- {:key :sync-and-analyze
- :db-schedule-column :metadata_sync_schedule
- :job-class SyncAndAnalyzeDatabase
- :name "Sync and Analyze"})
-
-(assert (mr/validate TaskInfo sync-analyze-task-info))
-
-(def ^:private field-values-task-info
- {:key :update-field-values
- :db-schedule-column :cache_field_values_schedule
- :job-class UpdateFieldValues
- :name "Scan Field Values"})
-
-(assert (mr/validate TaskInfo field-values-task-info))
-
-(def ^:private all-tasks
- [sync-analyze-task-info field-values-task-info])
-
-;; These getter functions are not strictly necessary but are provided primarily so we can get some extra validation by
-;; using them
-
-(mu/defn- job-key :- (ms/InstanceOfClass JobKey)
- "Return an appropriate string key for the job described by `task-info` for `database-or-id`."
- ^JobKey [task-info :- TaskInfo]
- (jobs/key (format "metabase.task.%s.job" (name (:key task-info)))))
-
-(mu/defn- trigger-key :- (ms/InstanceOfClass TriggerKey)
- "Return an appropriate string key for the trigger for `task-info` and `database-or-id`."
- ^TriggerKey [database :- (ms/InstanceOf :model/Database)
- task-info :- TaskInfo]
- (triggers/key (format "metabase.task.%s.trigger.%d" (name (:key task-info)) (u/the-id database))))
-
-(mu/defn- cron-schedule :- [:maybe u.cron/CronScheduleString]
- "Fetch the appropriate cron schedule string for `database` and `task-info`."
- [database :- (ms/InstanceOf :model/Database)
- task-info :- TaskInfo]
- (get database (:db-schedule-column task-info)))
-
-(mu/defn- job-class :- ::class
- "Get the Job class for `task-info`."
- [task-info :- TaskInfo]
- (:job-class task-info))
-
-(mu/defn- trigger-description :- :string
- "Return an appropriate description string for a job/trigger for Database described by `task-info`."
- [database :- (ms/InstanceOf :model/Database)
- task-info :- TaskInfo]
- (format "%s Database %d" (name (:key task-info)) (u/the-id database)))
-
-(mu/defn- job-description :- :string
- "Return an appropriate description string for a job"
- [task-info :- TaskInfo]
- (format "%s for all databases" (name (:key task-info))))
-
-;;; +----------------------------------------------------------------------------------------------------------------+
-;;; | DELETING TASKS FOR A DB |
-;;; +----------------------------------------------------------------------------------------------------------------+
-
-(mu/defn- delete-trigger!
- "Cancel a single sync trigger for `database-or-id` and `task-info`."
- [database :- (ms/InstanceOf :model/Database)
- task-info :- TaskInfo]
- (let [trigger-key (trigger-key database task-info)]
- (log/debug (u/format-color 'red
- (format "Unscheduling task for Database %d: trigger: %s" (u/the-id database) (.getName trigger-key))))
- (task/delete-trigger! trigger-key)))
-
-(mu/defn unschedule-tasks-for-db!
- "Cancel *all* scheduled sync and FieldValues caching tasks for `database-or-id`."
- [database :- (ms/InstanceOf :model/Database)]
- (doseq [task all-tasks]
- (delete-trigger! database task)))
-
-;;; +----------------------------------------------------------------------------------------------------------------+
-;;; | (RE)SCHEDULING TASKS FOR A DB |
-;;; +----------------------------------------------------------------------------------------------------------------+
-
(mu/defn- job :- (ms/InstanceOfClass JobDetail)
"Build a durable Quartz Job for `task-info`. Durable in Quartz allows the job to exist even if there are no triggers
for it."
- ^JobDetail [task-info :- TaskInfo]
+ ^JobDetail [task-info :- sync-databases-trigger/TaskInfo
+ job-class :- ::class]
(jobs/build
- (jobs/with-description (job-description task-info))
- (jobs/of-type (job-class task-info))
- (jobs/with-identity (job-key task-info))
+ (jobs/with-description (sync-databases-trigger/job-description task-info))
+ (jobs/of-type job-class)
+ (jobs/with-identity (sync-databases-trigger/job-key task-info))
(jobs/store-durably)))
-(def ^:private sync-analyze-job (job sync-analyze-task-info))
-(def ^:private field-values-job (job field-values-task-info))
-
-(mu/defn- trigger :- [:maybe (ms/InstanceOfClass CronTrigger)]
- "Build a Quartz Trigger for `database` and `task-info` if a schedule exists."
- ^CronTrigger [database :- (ms/InstanceOf :model/Database)
- task-info :- TaskInfo]
- (when-let [task-schedule (cron-schedule database task-info)]
- (triggers/build
- (triggers/with-description (trigger-description database task-info))
- (triggers/with-identity (trigger-key database task-info))
- (triggers/using-job-data {"db-id" (u/the-id database)})
- (triggers/for-job (job-key task-info))
- (triggers/start-now)
- (triggers/with-schedule
- (cron/schedule
- (cron/cron-schedule task-schedule)
- ;; if we miss a sync for one reason or another (such as system being down) do not try to run the sync again.
- ;; Just wait until the next sync cycle.
- ;;
- ;; See https://www.nurkiewicz.com/2012/04/quartz-scheduler-misfire-instructions.html for more info
- (cron/with-misfire-handling-instruction-do-nothing))))))
-
-(defn- update-db-trigger-if-needed!
- "Replace or remove the existing trigger if the schedule changes, do nothing if schedule is the same."
- [database task-info]
- (let [job (task/job-info (job-key task-info))
- new-trigger (trigger database task-info)
- ;; there should be only one schedule per task per DB
- existing-trigger-with-same-schedule (when new-trigger
- (let [trigger-key (.. new-trigger getKey getName)
- task-schedule (cron-schedule database task-info)]
- (some #(when (and (= (:key %) trigger-key)
- (= (:schedule %) task-schedule))
- %)
- (:triggers job))))]
- (cond
- ;; no new schedule
- ;; delete the existing trigger
- (nil? new-trigger)
- (do
- (log/infof "Trigger for \"%s\" of Database %d has been removed. It will no longer run on a schedule."
- (:name task-info)
- (u/the-id database))
- (delete-trigger! database task-info))
-
- ;; need to recreate the new trigger
- (and (some? new-trigger)
- (nil? existing-trigger-with-same-schedule))
- (do
- (if (delete-trigger! database task-info)
- (log/infof "Trigger for \"%s\" of Database %d has been updated. The new schedule is: \"%s\""
- (:name task-info)
- (u/the-id database)
- (cron-schedule database task-info))
- (log/infof "A trigger for \"%s\" of Database %d has been enabled with schedule: \"%s\""
- (:name task-info)
- (u/the-id database)
- (cron-schedule database task-info)))
- (task/add-trigger! new-trigger))
-
- ;; don't need to do anything as the existing trigger matches the new schedule
- :else
- nil)))
-
-;; called [[from metabase.warehouses.models.database/schedule-tasks!]] from the post-insert and the pre-update
-(mu/defn check-and-schedule-tasks-for-db!
- "Schedule a new Quartz job for `database` and `task-info` if it doesn't already exist or is incorrect."
- [database :- (ms/InstanceOf :model/Database)]
- (doseq [task all-tasks]
- (cond
- (:is_stub database)
- (log/info (u/format-color :red "Not scheduling sync task for stub database"))
-
- (and (= audit/audit-db-id (:id database))
- (= task sync-analyze-task-info))
- (log/info (u/format-color :red "Not scheduling sync task for audit database"))
-
- :else
- (update-db-trigger-if-needed! database task))))
+(def ^:private sync-analyze-job (job sync-databases-trigger/sync-analyze-task-info SyncAndAnalyzeDatabase))
+(def ^:private field-values-job (job sync-databases-trigger/field-values-task-info UpdateFieldValues))
;;; +----------------------------------------------------------------------------------------------------------------+
;;; | TASK INITIALIZATION |
diff --git a/src/metabase/sync/task/sync_databases_trigger.clj b/src/metabase/sync/task/sync_databases_trigger.clj
new file mode 100644
index 00000000000..369da9fbe2d
--- /dev/null
+++ b/src/metabase/sync/task/sync_databases_trigger.clj
@@ -0,0 +1,186 @@
+(ns metabase.sync.task.sync-databases-trigger
+ "Quartz trigger lifecycle for the per-Database sync-and-analyze and field-values tasks.
+
+ Separate from [[metabase.sync.task.sync-databases]] so that :model/Database lifecycle hooks can (un)schedule
+ triggers: the job bodies over there pull in the sync machinery, which reads the Database model."
+ (:require
+ [clojurewerkz.quartzite.jobs :as jobs]
+ [clojurewerkz.quartzite.schedule.cron :as cron]
+ [clojurewerkz.quartzite.triggers :as triggers]
+ [metabase.audit-app.core :as audit]
+ [metabase.task.core :as task]
+ [metabase.util :as u]
+ [metabase.util.cron :as u.cron]
+ [metabase.util.log :as log]
+ [metabase.util.malli :as mu]
+ [metabase.util.malli.registry :as mr]
+ [metabase.util.malli.schema :as ms])
+ (:import
+ (org.quartz
+ CronTrigger
+ JobKey
+ TriggerKey)))
+
+(set! *warn-on-reflection* true)
+
+;;; +----------------------------------------------------------------------------------------------------------------+
+;;; | TASK INFO AND GETTER FUNCTIONS |
+;;; +----------------------------------------------------------------------------------------------------------------+
+
+(def TaskInfo
+ "One-off schema for information about the various sync tasks we run for a DB."
+ [:map
+ [:key :keyword]
+ [:db-schedule-column :keyword]
+ [:name :string]])
+
+(def sync-analyze-task-info
+ "Task info for the sync-and-analyze task."
+ {:key :sync-and-analyze
+ :db-schedule-column :metadata_sync_schedule
+ :name "Sync and Analyze"})
+
+(assert (mr/validate TaskInfo sync-analyze-task-info))
+
+(def field-values-task-info
+ "Task info for the Field values caching task."
+ {:key :update-field-values
+ :db-schedule-column :cache_field_values_schedule
+ :name "Scan Field Values"})
+
+(assert (mr/validate TaskInfo field-values-task-info))
+
+(def ^:private all-tasks
+ [sync-analyze-task-info field-values-task-info])
+
+;; These getter functions are not strictly necessary but are provided primarily so we can get some extra validation by
+;; using them
+
+(mu/defn job-key :- (ms/InstanceOfClass JobKey)
+ "Return an appropriate string key for the job described by `task-info` for `database-or-id`."
+ ^JobKey [task-info :- TaskInfo]
+ (jobs/key (format "metabase.task.%s.job" (name (:key task-info)))))
+
+(mu/defn- trigger-key :- (ms/InstanceOfClass TriggerKey)
+ "Return an appropriate string key for the trigger for `task-info` and `database-or-id`."
+ ^TriggerKey [database :- (ms/InstanceOf :model/Database)
+ task-info :- TaskInfo]
+ (triggers/key (format "metabase.task.%s.trigger.%d" (name (:key task-info)) (u/the-id database))))
+
+(mu/defn- cron-schedule :- [:maybe u.cron/CronScheduleString]
+ "Fetch the appropriate cron schedule string for `database` and `task-info`."
+ [database :- (ms/InstanceOf :model/Database)
+ task-info :- TaskInfo]
+ (get database (:db-schedule-column task-info)))
+
+(mu/defn- trigger-description :- :string
+ "Return an appropriate description string for a job/trigger for Database described by `task-info`."
+ [database :- (ms/InstanceOf :model/Database)
+ task-info :- TaskInfo]
+ (format "%s Database %d" (name (:key task-info)) (u/the-id database)))
+
+(mu/defn job-description :- :string
+ "Return an appropriate description string for a job"
+ [task-info :- TaskInfo]
+ (format "%s for all databases" (name (:key task-info))))
+
+;;; +----------------------------------------------------------------------------------------------------------------+
+;;; | DELETING TASKS FOR A DB |
+;;; +----------------------------------------------------------------------------------------------------------------+
+
+(mu/defn- delete-trigger!
+ "Cancel a single sync trigger for `database-or-id` and `task-info`."
+ [database :- (ms/InstanceOf :model/Database)
+ task-info :- TaskInfo]
+ (let [trigger-key (trigger-key database task-info)]
+ (log/debug (u/format-color 'red
+ (format "Unscheduling task for Database %d: trigger: %s" (u/the-id database) (.getName trigger-key))))
+ (task/delete-trigger! trigger-key)))
+
+(mu/defn unschedule-tasks-for-db!
+ "Cancel *all* scheduled sync and FieldValues caching tasks for `database-or-id`."
+ [database :- (ms/InstanceOf :model/Database)]
+ (doseq [task all-tasks]
+ (delete-trigger! database task)))
+
+;;; +----------------------------------------------------------------------------------------------------------------+
+;;; | (RE)SCHEDULING TASKS FOR A DB |
+;;; +----------------------------------------------------------------------------------------------------------------+
+
+(mu/defn- trigger :- [:maybe (ms/InstanceOfClass CronTrigger)]
+ "Build a Quartz Trigger for `database` and `task-info` if a schedule exists."
+ ^CronTrigger [database :- (ms/InstanceOf :model/Database)
+ task-info :- TaskInfo]
+ (when-let [task-schedule (cron-schedule database task-info)]
+ (triggers/build
+ (triggers/with-description (trigger-description database task-info))
+ (triggers/with-identity (trigger-key database task-info))
+ (triggers/using-job-data {"db-id" (u/the-id database)})
+ (triggers/for-job (job-key task-info))
+ (triggers/start-now)
+ (triggers/with-schedule
+ (cron/schedule
+ (cron/cron-schedule task-schedule)
+ ;; if we miss a sync for one reason or another (such as system being down) do not try to run the sync again.
+ ;; Just wait until the next sync cycle.
+ ;;
+ ;; See https://www.nurkiewicz.com/2012/04/quartz-scheduler-misfire-instructions.html for more info
+ (cron/with-misfire-handling-instruction-do-nothing))))))
+
+(defn- update-db-trigger-if-needed!
+ "Replace or remove the existing trigger if the schedule changes, do nothing if schedule is the same."
+ [database task-info]
+ (let [job (task/job-info (job-key task-info))
+ new-trigger (trigger database task-info)
+ ;; there should be only one schedule per task per DB
+ existing-trigger-with-same-schedule (when new-trigger
+ (let [trigger-key (.. new-trigger getKey getName)
+ task-schedule (cron-schedule database task-info)]
+ (some #(when (and (= (:key %) trigger-key)
+ (= (:schedule %) task-schedule))
+ %)
+ (:triggers job))))]
+ (cond
+ ;; no new schedule
+ ;; delete the existing trigger
+ (nil? new-trigger)
+ (do
+ (log/infof "Trigger for \"%s\" of Database %d has been removed. It will no longer run on a schedule."
+ (:name task-info)
+ (u/the-id database))
+ (delete-trigger! database task-info))
+
+ ;; need to recreate the new trigger
+ (and (some? new-trigger)
+ (nil? existing-trigger-with-same-schedule))
+ (do
+ (if (delete-trigger! database task-info)
+ (log/infof "Trigger for \"%s\" of Database %d has been updated. The new schedule is: \"%s\""
+ (:name task-info)
+ (u/the-id database)
+ (cron-schedule database task-info))
+ (log/infof "A trigger for \"%s\" of Database %d has been enabled with schedule: \"%s\""
+ (:name task-info)
+ (u/the-id database)
+ (cron-schedule database task-info)))
+ (task/add-trigger! new-trigger))
+
+ ;; don't need to do anything as the existing trigger matches the new schedule
+ :else
+ nil)))
+
+;; called [[from metabase.warehouses.models.database/schedule-tasks!]] from the post-insert and the pre-update
+(mu/defn check-and-schedule-tasks-for-db!
+ "Schedule a new Quartz job for `database` and `task-info` if it doesn't already exist or is incorrect."
+ [database :- (ms/InstanceOf :model/Database)]
+ (doseq [task all-tasks]
+ (cond
+ (:is_stub database)
+ (log/info (u/format-color :red "Not scheduling sync task for stub database"))
+
+ (and (= audit/audit-db-id (:id database))
+ (= task sync-analyze-task-info))
+ (log/info (u/format-color :red "Not scheduling sync task for audit database"))
+
+ :else
+ (update-db-trigger-if-needed! database task))))
diff --git a/src/metabase/task/QUARTZ.md b/src/metabase/task/QUARTZ.md
index 9562a9bfe41..dc7120485ae 100644
--- a/src/metabase/task/QUARTZ.md
+++ b/src/metabase/task/QUARTZ.md
@@ -84,7 +84,7 @@ These tasks run on a recurring schedule defined by a cron expression.
Notice in the example above that we set the timezone explicitly with `(cron/in-time-zone "UTC")`. By default, Quartz schedules are interpreted in the JVM's default timezone, but in Metabase we usually set the timezone to `report-timezone` .
-**Important Note:** If your tasks are timezone sensitive, you'll need to rebuild the triggers when `driver/report-timezone` changes. See [[metabase.notification.task.send/update-send-notification-triggers-timezone!]] as an example of how we handle rescheduling.
+**Important Note:** If your tasks are timezone sensitive, you'll need to rebuild the triggers when `driver/report-timezone` changes. See [[metabase.notification.task.send-trigger/update-send-notification-triggers-timezone!]] as an example of how we handle rescheduling.
### 3. Tasks that run once like a migration
diff --git a/src/metabase/task/bootstrap.clj b/src/metabase/task/bootstrap.clj
index 1cdfa48f00d..4cec974e70d 100644
--- a/src/metabase/task/bootstrap.clj
+++ b/src/metabase/task/bootstrap.clj
@@ -6,12 +6,6 @@
(set! *warn-on-reflection* true)
-;; Custom `ConnectionProvider` implementation that uses a dedicated connection pool for the application DB to provide
-;; connections.
-
-(defn- quartz-data-source ^javax.sql.DataSource []
- ((requiring-resolve 'metabase.app-db.core/quartz-data-source)))
-
;; Optional interceptor for wrapping JDBC connections before Quartz uses them.
;; Set by task.tracing to add SQL-level tracing. nil means no interception.
(defonce ^:private connection-interceptor (atom nil))
@@ -22,28 +16,12 @@
[f]
(reset! connection-interceptor f))
-(defrecord ^:private ConnectionProvider []
- org.quartz.utils.ConnectionProvider
- (initialize [_])
- (getConnection [_]
- ;; get a connection from the dedicated Quartz connection pool. Quartz will close it (i.e., return it to the pool)
- ;; when it's done.
- ;;
- ;; very important! Fetch a new connection from the connection pool rather than reusing a Connection already bound
- ;; to the calling thread (e.g. toucan2's *current-connectable*) -- Quartz manages the connection's whole
- ;; lifecycle (setAutoCommit/commit/rollback/close), and its cluster locking relies on commit/rollback to release
- ;; row locks on the QRTZ_LOCKS table, so it must never share a connection with an outer transaction.
- ;;
- ;; the pool is separate from the main application DB pool so that a Quartz operation triggered by a thread inside
- ;; a `with-transaction` block can't deadlock when application code has saturated the main pool.
- (let [conn (.getConnection (quartz-data-source))]
- (if-let [interceptor @connection-interceptor]
- (interceptor conn)
- conn)))
- (shutdown [_]))
-
-(when-not *compile-files*
- (System/setProperty "org.quartz.dataSource.db.connectionProvider.class" (.getName ConnectionProvider)))
+(defn intercept-connection
+ "Wrap `conn` with the interceptor set by [[set-connection-interceptor!]], if any."
+ [conn]
+ (if-let [interceptor @connection-interceptor]
+ (interceptor conn)
+ conn))
(defn- load-class ^Class [^String class-name]
(Class/forName class-name true (classloader/the-classloader)))
diff --git a/src/metabase/tracing/quartz.clj b/src/metabase/tracing/quartz.clj
index 4f333a3f640..e1d93110d4c 100644
--- a/src/metabase/tracing/quartz.clj
+++ b/src/metabase/tracing/quartz.clj
@@ -136,7 +136,7 @@
(invoke-or-unwrap method conn args)))))))
(defn- connection-interceptor
- "Connection wrapping function installed into bootstrap's ConnectionProvider.
+ "Connection wrapping function installed into the app-db's Quartz ConnectionProvider.
Checks `:quartz` group at call time — wraps only when enabled."
[^Connection conn]
(if (tracing/group-enabled? :quartz)
diff --git a/src/metabase/warehouse_schema/models/field_values.clj b/src/metabase/warehouse_schema/models/field_values.clj
index 84dfeec92be..1e61496b200 100644
--- a/src/metabase/warehouse_schema/models/field_values.clj
+++ b/src/metabase/warehouse_schema/models/field_values.clj
@@ -42,6 +42,7 @@
[metabase.util.malli.registry :as mr]
[metabase.util.malli.schema :as ms]
[metabase.warehouse-schema.db :as warehouse-schema.db]
+ [metabase.warehouse-schema.metadata-from-qp :as metadata-from-qp]
[methodical.core :as methodical]
[toucan2.core :as t2]))
@@ -387,7 +388,7 @@
(let [field (cond-> field
;; a caller may hand us a Field hydrated with its Table, which is not part of column metadata
(t2/model field) (-> (dissoc :table) (lib-be/instance->metadata :metadata/column)))
- result ((requiring-resolve 'metabase.warehouse-schema.metadata-from-qp/table-query)
+ result (metadata-from-qp/table-query
(:table-id field)
(fn [query]
(-> query
diff --git a/src/metabase/warehouses/models/database.clj b/src/metabase/warehouses/models/database.clj
index bf79193e94b..83d4c463837 100644
--- a/src/metabase/warehouses/models/database.clj
+++ b/src/metabase/warehouses/models/database.clj
@@ -16,11 +16,12 @@
[metabase.models.serialization :as serdes]
[metabase.permissions.core :as perms]
[metabase.premium-features.core :as premium-features :refer [defenterprise]]
- ;; Trying to use metabase.search would cause a circular reference ;_;
+ [metabase.search.core :as search]
[metabase.search.spec :as search.spec]
[metabase.secrets.core :as secret]
[metabase.settings.core :as setting]
[metabase.sync.schedules :as sync.schedules]
+ [metabase.sync.task.sync-databases-trigger :as sync-databases-trigger]
[metabase.util :as u]
[metabase.util.honey-sql-2 :as h2x]
[metabase.util.i18n :refer [trs tru]]
@@ -238,9 +239,8 @@
"(Re)schedule sync operation tasks for `database`. (Existing scheduled tasks will be deleted first.)"
[database]
(try
- ;; this is done this way to avoid circular dependencies
(when (should-auto-sync? database)
- ((requiring-resolve 'metabase.sync.task.sync-databases/check-and-schedule-tasks-for-db!) database))
+ (sync-databases-trigger/check-and-schedule-tasks-for-db! database))
(catch Throwable e
(log/errorf "Error scheduling tasks for DB: %s" (ex-message e)))))
@@ -362,7 +362,7 @@
"Unschedule any currently pending sync operation tasks for `database`."
[database]
(try
- ((requiring-resolve 'metabase.sync.task.sync-databases/unschedule-tasks-for-db!) database)
+ (sync-databases-trigger/unschedule-tasks-for-db! database)
(catch Throwable e
(log/errorf "Error unscheduling tasks for DB: %s" (ex-message e)))))
@@ -447,8 +447,7 @@
(warehouses.db/delete-cards-for-database-returning-ids-reducible id)
(warehouses.db/card-ids-for-database-reducible id)))
(run! (fn [batch]
- ;; damn circular deps
- ((requiring-resolve 'metabase.search.core/delete!) :model/Card (map (comp str :id) batch)))))
+ (search/delete! :model/Card (map (comp str :id) batch)))))
(when (not= :postgres (mdb/db-type))
(warehouses.db/delete-cards-for-database! id))
(try
diff --git a/test/metabase/app_db/custom_migrations_test.clj b/test/metabase/app_db/custom_migrations_test.clj
index cb14136ed69..2e72908d3a9 100644
--- a/test/metabase/app_db/custom_migrations_test.clj
+++ b/test/metabase/app_db/custom_migrations_test.clj
@@ -31,6 +31,7 @@
[metabase.permissions.models.permissions-group :as perms-group]
[metabase.pulse.models.pulse-channel-test :as pulse-channel-test]
[metabase.pulse.task.send-pulses :as task.send-pulses]
+ [metabase.pulse.task.send-pulses-trigger :as task.send-pulses-trigger]
[metabase.search.ingestion :as search.ingestion]
[metabase.settings.models.setting :as setting]
[metabase.sync.task.sync-databases-test :as task.sync-databases-test]
@@ -1809,7 +1810,7 @@
pulse-id (:id (new-instance-with-default :pulse {:creator_id user-id}))
pc (new-instance-with-default :pulse_channel {:pulse_id pulse-id})]
;; trigger this so we schedule a trigger for send-pulse
- (task.send-pulses/update-send-pulse-trigger-if-needed! pulse-id pc :add-pc-ids #{(:id pc)})
+ (task.send-pulses-trigger/update-send-pulse-trigger-if-needed! pulse-id pc :add-pc-ids #{(:id pc)})
(testing "sanity check that we have a send pulse trigger and 2 jobs"
(is (= 1 (count (pulse-channel-test/send-pulse-triggers pulse-id))))
(is (= #{"metabase.task.send-pulses.send-pulse.job"
diff --git a/test/metabase/task/bootstrap_test.clj b/test/metabase/app_db/quartz_test.clj
similarity index 94%
rename from test/metabase/task/bootstrap_test.clj
rename to test/metabase/app_db/quartz_test.clj
index 3441118412b..e129db54bfa 100644
--- a/test/metabase/task/bootstrap_test.clj
+++ b/test/metabase/app_db/quartz_test.clj
@@ -1,10 +1,10 @@
-(ns metabase.task.bootstrap-test
+(ns metabase.app-db.quartz-test
(:require
[clojure.test :refer :all]
[metabase.app-db.connection :as mdb.connection]
[metabase.app-db.connection-pool-setup :as mdb.connection-pool-setup]
[metabase.app-db.data-source :as mdb.data-source]
- [metabase.task.bootstrap :as task.bootstrap]
+ [metabase.app-db.quartz :as mdb.quartz]
[toucan2.connection :as t2.conn])
(:import
(com.mchange.v2.c3p0 DataSources)
@@ -23,7 +23,7 @@
(binding [mdb.connection/*application-db* app-db]
(with-open [^Connection outer (.getConnection data-source)]
(binding [t2.conn/*current-connectable* outer]
- (let [^ConnectionProvider provider (task.bootstrap/->ConnectionProvider)]
+ (let [^ConnectionProvider provider (mdb.quartz/->ConnectionProvider)]
(with-open [^Connection conn (.getConnection provider)]
(is (instance? Connection conn))
(is (not (identical? outer conn))
@@ -56,7 +56,7 @@
(with-open [^Connection main-conn (.getConnection ^javax.sql.DataSource app-db)]
;; ...and simulate being inside a `with-transaction` block on it
(binding [t2.conn/*current-connectable* main-conn]
- (let [^ConnectionProvider provider (task.bootstrap/->ConnectionProvider)
+ (let [^ConnectionProvider provider (mdb.quartz/->ConnectionProvider)
result (future
(with-open [^Connection conn (.getConnection provider)]
(with-open [stmt (.createStatement conn)]
diff --git a/test/metabase/indexed_entities/task/index_values_test.clj b/test/metabase/indexed_entities/task/index_values_test.clj
index 5650bb5e990..c05f9657318 100644
--- a/test/metabase/indexed_entities/task/index_values_test.clj
+++ b/test/metabase/indexed_entities/task/index_values_test.clj
@@ -2,6 +2,7 @@
{:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.indexed-entities.task.index-values-test]}}}}}}
(:require
[clojure.test :refer :all]
+ [metabase.indexed-entities.models.model-index :as model-index]
[metabase.indexed-entities.models.model-index-test :refer [with-scheduler-setup!]]
[metabase.indexed-entities.task.index-values :as task.index-values]
[metabase.task.impl :as task]
@@ -36,7 +37,7 @@
(task.index-values/add-indexing-job model-index)
(is (some? (get-trigger)) "Trigger should exist after creating model index")
;; delete the trigger from Quartz
- (task.index-values/remove-indexing-job model-index)
+ (model-index/remove-indexing-job model-index)
(is (nil? (get-trigger)) "Trigger should be deleted")
;; call job-init! to recreate missing triggers
(#'task.index-values/job-init!)
diff --git a/test/metabase/notification/task/send_test.clj b/test/metabase/notification/task/send_test.clj
index 4228a0cd0c2..a254a6b245c 100644
--- a/test/metabase/notification/task/send_test.clj
+++ b/test/metabase/notification/task/send_test.clj
@@ -5,6 +5,7 @@
[metabase.driver :as driver]
[metabase.notification.models :as models.notification]
[metabase.notification.task.send :as task.notification]
+ [metabase.notification.task.send-trigger :as notification.task.send-trigger]
[metabase.notification.test-util :as notification.tu]
[metabase.task.core :as task]
[metabase.test :as mt]
@@ -66,13 +67,13 @@
(testing "sanity check that it has triggers to begin with"
(is (not-empty notification-triggers)))
(testing "init send notification triggers are idempotent if the subscription doesn't change"
- (task.notification/init-send-notification-triggers!)
+ (notification.task.send-trigger/init-send-notification-triggers!)
(is (= notification-triggers (notification.tu/send-notification-triggers subscription-id))))
(testing "Re-create triggers if it's not existed"
(task/delete-trigger! (TriggerKey. (-> notification-triggers first :key)))
(testing "sanity check that the trigger is deleted"
(is (empty? (notification.tu/send-notification-triggers subscription-id))))
- (task.notification/init-send-notification-triggers!)
+ (notification.task.send-trigger/init-send-notification-triggers!)
(is (= notification-triggers (notification.tu/send-notification-triggers subscription-id))))
(testing "deletes triggers for subscriptions that no longer exist"
(let [subscription-id (first (t2/select-pks-vec :model/NotificationSubscription
@@ -80,7 +81,7 @@
(t2/delete! :notification_subscription subscription-id)
(testing "sanity check that it has trigger before"
(is (not-empty (notification.tu/send-notification-triggers subscription-id))))
- (task.notification/init-send-notification-triggers!)
+ (notification.task.send-trigger/init-send-notification-triggers!)
(is (empty? (notification.tu/send-notification-triggers subscription-id)))))))))
(deftest init-send-notification-triggers-inactive-test
@@ -100,12 +101,12 @@
(testing "skips triggers for inactive notifications"
;; Deactivate the notification
(t2/update! :model/Notification (:id notification) {:active false})
- (task.notification/init-send-notification-triggers!)
+ (notification.task.send-trigger/init-send-notification-triggers!)
(is (empty? (notification.tu/send-notification-triggers subscription-id))))
(testing "recreates triggers when notification is reactivated"
;; Reactivate the notification
(t2/update! :model/Notification (:id notification) {:active true})
- (task.notification/init-send-notification-triggers!)
+ (notification.task.send-trigger/init-send-notification-triggers!)
(is (= notification-triggers (notification.tu/send-notification-triggers subscription-id))))))))
(deftest update-send-notification-triggers-timezone-test
@@ -128,7 +129,7 @@
;; Change the report timezone
(driver/report-timezone! new-timezone)
;; Call the function under test
- (task.notification/update-send-notification-triggers-timezone!)
+ (notification.task.send-trigger/update-send-notification-triggers-timezone!)
;; Verify the timezone was updated
(let [updated-triggers (notification.tu/send-notification-triggers subscription-id)]
(is (not-empty updated-triggers))
diff --git a/test/metabase/notification/test_util.clj b/test/metabase/notification/test_util.clj
index a8a8af06fd4..965ccadab5f 100644
--- a/test/metabase/notification/test_util.clj
+++ b/test/metabase/notification/test_util.clj
@@ -15,7 +15,7 @@
[metabase.notification.models :as models.notification]
[metabase.notification.payload.core :as notification.payload]
[metabase.notification.send :as notification.send]
- [metabase.notification.task.send :as task.notification]
+ [metabase.notification.task.send-trigger :as notification.task.send-trigger]
[metabase.task.core :as task]
[metabase.test :as mt]
[metabase.util :as u]
@@ -261,8 +261,8 @@
[subscription-id]
(map
#(select-keys % [:key :schedule :data :timezone])
- (task/existing-triggers @#'task.notification/send-notification-job-key
- (#'task.notification/send-notification-trigger-key subscription-id))))
+ (task/existing-triggers notification.task.send-trigger/send-notification-job-key
+ (#'notification.task.send-trigger/send-notification-trigger-key subscription-id))))
(defn notification-triggers
"Return the quartz triggers for a notification."
@@ -275,7 +275,7 @@
([subscription-id cron-schedule]
(subscription->trigger-info subscription-id cron-schedule "UTC"))
([subscription-id cron-schedule timezone]
- {:key (.getName ^org.quartz.TriggerKey (#'task.notification/send-notification-trigger-key subscription-id))
+ {:key (.getName ^org.quartz.TriggerKey (#'notification.task.send-trigger/send-notification-trigger-key subscription-id))
:schedule cron-schedule
:data {"subscription-id" subscription-id}
:timezone timezone}))
diff --git a/test/metabase/pulse/models/pulse_channel_test.clj b/test/metabase/pulse/models/pulse_channel_test.clj
index c01bcaadfca..40f313d2423 100644
--- a/test/metabase/pulse/models/pulse_channel_test.clj
+++ b/test/metabase/pulse/models/pulse_channel_test.clj
@@ -4,6 +4,7 @@
[medley.core :as m]
[metabase.pulse.models.pulse-channel :as pulse-channel]
[metabase.pulse.task.send-pulses :as task.send-pulses]
+ [metabase.pulse.task.send-pulses-trigger :as task.send-pulses-trigger]
[metabase.task.core :as task]
[metabase.test :as mt]
[metabase.util :as u]
@@ -356,7 +357,7 @@
(defn pulse->trigger-info
[pulse-id schedule-map pc-ids]
- {:key (.getName ^org.quartz.TriggerKey (#'task.send-pulses/send-pulse-trigger-key pulse-id schedule-map))
+ {:key (.getName ^org.quartz.TriggerKey (#'task.send-pulses-trigger/send-pulse-trigger-key pulse-id schedule-map))
:schedule (u.cron/schedule-map->cron-string schedule-map)
:priority 6
:data {"pulse-id" pulse-id
@@ -376,7 +377,7 @@
(defn send-pulse-triggers
[pulse-id & {:keys [additional-keys]}]
- (->> (task/job-info @#'task.send-pulses/send-pulse-job-key)
+ (->> (task/job-info task.send-pulses-trigger/send-pulse-job-key)
:triggers
(map #(select-keys % (concat [:key :schedule :data :priority] additional-keys)))
(filter #(or (nil? pulse-id) (= pulse-id (get-in % [:data "pulse-id"]))))
diff --git a/test/metabase/pulse/task/send_pulses_test.clj b/test/metabase/pulse/task/send_pulses_test.clj
index 4b7c3abbca6..5eab7f21ccb 100644
--- a/test/metabase/pulse/task/send_pulses_test.clj
+++ b/test/metabase/pulse/task/send_pulses_test.clj
@@ -9,6 +9,7 @@
[metabase.pulse.models.pulse-channel-test :as pulse-channel-test]
[metabase.pulse.send :as pulse.send]
[metabase.pulse.task.send-pulses :as task.send-pulses]
+ [metabase.pulse.task.send-pulses-trigger :as task.send-pulses-trigger]
[metabase.task.core :as task]
[metabase.test :as mt]
[metabase.test.util :as mt.util]
@@ -189,7 +190,7 @@
(testing "sanity check that there are no triggers"
(is (empty? (all-send-pulse-triggers))))
(testing "init-send-pulse-triggers! should create triggers for each pulse-channel"
- (#'task.send-pulses/init-dashboard-subscription-triggers!)
+ (#'task.send-pulses-trigger/init-dashboard-subscription-triggers!)
(is (=? #{(pulse-channel-test/pulse->trigger-info pulse-1 daily-at-1am [pc-1-1 pc-1-2])
;; pc-2-1 has the same schedule as pc-1-1 and pc-1-2 but it's not on the same trigger because it's a
;; different schedule
@@ -354,7 +355,7 @@
(testing "sanity check that it has triggers to begin with"
(is (not-empty pulse-triggers)))
(testing "init send pulse triggers are idempotent if the pulse channel doesn't change"
- (#'task.send-pulses/init-dashboard-subscription-triggers!)
+ (#'task.send-pulses-trigger/init-dashboard-subscription-triggers!)
(is (= pulse-triggers (pulse-channel-test/send-pulse-triggers pulse-id)))))))))
(deftest init-send-pulse-triggers-skip-alert-test
@@ -368,4 +369,4 @@
:channel_type :slack
:details {:channel "#random"}}
daily-at-1am)]
- (is (not (contains? (set (map :id (#'task.send-pulses/active-dashsub-pcs))) pc-id)))))))
+ (is (not (contains? (set (map :id (#'task.send-pulses-trigger/active-dashsub-pcs))) pc-id)))))))
diff --git a/test/metabase/queries/models/card_test.clj b/test/metabase/queries/models/card_test.clj
index 9f65ceee28e..12afa0cd172 100644
--- a/test/metabase/queries/models/card_test.clj
+++ b/test/metabase/queries/models/card_test.clj
@@ -17,6 +17,7 @@
[metabase.queries.models.card :as card]
[metabase.queries.models.parameter-card :as parameter-card]
[metabase.queries.schema :as queries.schema]
+ [metabase.query-processor.card :as qp.card]
[metabase.query-processor.card-test :as qp.card-test]
[metabase.query-processor.preprocess :as qp.preprocess]
[metabase.search.ingestion :as search.ingestion]
@@ -352,7 +353,7 @@
:slug "date"
:default nil
:required false}]
- (card/template-tag-parameters card))))))
+ (qp.card/template-tag-parameters card))))))
(deftest ^:parallel template-tag-parameters-test-2
(testing "Card with a non-Field-filter parameter"
@@ -364,7 +365,7 @@
:slug "id"
:default "1"
:required true}]
- (card/template-tag-parameters card))))))
+ (qp.card/template-tag-parameters card))))))
(deftest ^:parallel template-tag-parameters-test-3
(testing "Should ignore native query snippets and source card IDs"
@@ -376,7 +377,7 @@
:slug "id"
:default "1"
:required true}]
- (card/template-tag-parameters card))))))
+ (qp.card/template-tag-parameters card))))))
(defn- native-query-card
"Build a card map with a native query containing the given template tags.
@@ -403,7 +404,7 @@
:slug "name"
:default "Alice"
:required false}]
- (card/template-tag-parameters card))))))
+ (qp.card/template-tag-parameters card))))))
(deftest ^:parallel template-tag-parameters-boolean-tag-test
(testing ":boolean template tag without widget-type produces :boolean/= parameter type (QUE2-326)"
@@ -421,7 +422,7 @@
:slug "active"
:default nil
:required false}]
- (card/template-tag-parameters card))))))
+ (qp.card/template-tag-parameters card))))))
(deftest ^:parallel template-tag-parameters-dimension-category-widget-test
(testing ":dimension template tag with :category widget-type passes through widget-type (QUE2-326)"
@@ -441,7 +442,7 @@
:slug "cat"
:default nil
:required false}]
- (card/template-tag-parameters card))))))
+ (qp.card/template-tag-parameters card))))))
(deftest validate-template-tag-field-ids-test
(testing "Disallow saving a Card with native query Field filter template tags referencing a different Database (#14145)"
diff --git a/test/metabase/sso/api/ldap_test.clj b/test/metabase/sso/api/ldap_test.clj
index 74983caef80..8e4f37b2c96 100644
--- a/test/metabase/sso/api/ldap_test.clj
+++ b/test/metabase/sso/api/ldap_test.clj
@@ -5,6 +5,7 @@
[metabase.settings.core :as setting]
[metabase.sso.ldap :as ldap]
[metabase.sso.ldap-test-util :as ldap.test]
+ [metabase.sso.ldap.settings :as sso.ldap.settings]
[metabase.sso.settings :as sso.settings]
[metabase.test :as mt]))
@@ -52,7 +53,7 @@
:ldap-port (int (ldap.test/get-ldap-port)))))
(testing "Passing ldap-enabled=false will disable LDAP"
(mt/user-http-request :crowberto :put 200 "ldap/settings" (ldap-test-details false))
- (is (not (sso.settings/ldap-enabled))))
+ (is (not (sso.ldap.settings/ldap-enabled))))
(testing "Passing ldap-enabled=false still validates the LDAP settings"
(mt/user-http-request :crowberto :put 500 "ldap/settings"
(assoc (ldap-test-details false) :ldap-password "wrong-password")))
@@ -74,8 +75,8 @@
(testing "PUT /api/ldap/settings clearing the settings disables LDAP and clears the host"
(ldap.test/with-ldap-server!
(mt/user-http-request :crowberto :put 200 "ldap/settings" (ldap-test-details))
- (is (sso.settings/ldap-enabled))
+ (is (sso.ldap.settings/ldap-enabled))
(mt/with-dynamic-fn-redefs [ldap/test-ldap-connection (constantly {:status :SUCCESS})]
(mt/user-http-request :crowberto :put 200 "ldap/settings" {:ldap-host nil :ldap-enabled false}))
- (is (not (sso.settings/ldap-enabled)))
+ (is (not (sso.ldap.settings/ldap-enabled)))
(is (nil? (sso.settings/ldap-host))))))
diff --git a/test/metabase/sso/settings_test.clj b/test/metabase/sso/settings_test.clj
index 4b66e980a38..2c3bf1d3d56 100644
--- a/test/metabase/sso/settings_test.clj
+++ b/test/metabase/sso/settings_test.clj
@@ -3,6 +3,7 @@
[clojure.test :refer :all]
[metabase.sso.ldap :as ldap]
[metabase.sso.ldap-test-util :as ldap.test]
+ [metabase.sso.ldap.settings :as sso.ldap.settings]
[metabase.sso.settings :as sso.settings]
[metabase.test :as mt]))
@@ -13,12 +14,12 @@
(mt/with-dynamic-fn-redefs [ldap/test-current-ldap-details (constantly {:status :ERROR :message "test error"})]
(is (thrown-with-msg? clojure.lang.ExceptionInfo
#"Unable to connect to LDAP server"
- (sso.settings/ldap-enabled! true))))
+ (sso.ldap.settings/ldap-enabled! true))))
(mt/with-dynamic-fn-redefs [ldap/test-current-ldap-details (constantly {:status :SUCCESS})]
- (sso.settings/ldap-enabled! true)
- (is (sso.settings/ldap-enabled))
- (sso.settings/ldap-enabled! false)
- (is (not (sso.settings/ldap-enabled))))))))
+ (sso.ldap.settings/ldap-enabled! true)
+ (is (sso.ldap.settings/ldap-enabled))
+ (sso.ldap.settings/ldap-enabled! false)
+ (is (not (sso.ldap.settings/ldap-enabled))))))))
(deftest ^:parallel send-new-sso-user-admin-email?-test
(is ((some-fn nil? boolean?) (sso.settings/send-new-sso-user-admin-email?))
diff --git a/test/metabase/sync/task/sync_databases_test.clj b/test/metabase/sync/task/sync_databases_test.clj
index 0180b326ab5..b62b65c7d0e 100644
--- a/test/metabase/sync/task/sync_databases_test.clj
+++ b/test/metabase/sync/task/sync_databases_test.clj
@@ -10,6 +10,7 @@
[metabase.sync.field-values :as sync.field-values]
[metabase.sync.schedules :as sync.schedules]
[metabase.sync.task.sync-databases :as task.sync-databases]
+ [metabase.sync.task.sync-databases-trigger :as sync-databases-trigger]
[metabase.task.core :as task]
[metabase.test :as mt]
[metabase.test.util :as tu]
@@ -44,8 +45,8 @@
"Returns the name of trigger for DB.
These are all the trigger names that a database SHOULD have."
[db]
- (set (map #(.getName ^TriggerKey (#'task.sync-databases/trigger-key (t2/instance :model/Database db) %))
- @#'task.sync-databases/all-tasks)))
+ (set (map #(.getName ^TriggerKey (#'sync-databases-trigger/trigger-key (t2/instance :model/Database db) %))
+ @#'sync-databases-trigger/all-tasks)))
(defn query-all-db-sync-triggers-name
"Find the all triggers for DB \"db\".
@@ -53,10 +54,10 @@
[db]
(let [db (t2/instance :model/Database db)]
(assert (some? (#'task/scheduler)) "makes sure the scheduler is initialized!")
- (->> (for [task-info @#'task.sync-databases/all-tasks]
- (keep #(when (= (.getName ^TriggerKey (#'task.sync-databases/trigger-key db task-info)) (:key %))
+ (->> (for [task-info @#'sync-databases-trigger/all-tasks]
+ (keep #(when (= (.getName ^TriggerKey (#'sync-databases-trigger/trigger-key db task-info)) (:key %))
(:key %))
- (:triggers (task/job-info (#'task.sync-databases/job-key task-info)))))
+ (:triggers (task/job-info (#'sync-databases-trigger/job-key task-info)))))
flatten
set)))
@@ -221,14 +222,14 @@
(mt/with-temp [:model/Database non-stub {:is_stub false}
:model/Database stub {:is_stub true}]
(let [calls (atom 0)]
- (with-redefs [task.sync-databases/update-db-trigger-if-needed! (fn [_ _] (swap! calls inc))]
+ (with-redefs [sync-databases-trigger/update-db-trigger-if-needed! (fn [_ _] (swap! calls inc))]
(testing "non-stub: triggers are considered for scheduling"
(reset! calls 0)
- (task.sync-databases/check-and-schedule-tasks-for-db! non-stub)
+ (sync-databases-trigger/check-and-schedule-tasks-for-db! non-stub)
(is (pos? @calls)))
(testing "stub: no scheduling calls are made"
(reset! calls 0)
- (task.sync-databases/check-and-schedule-tasks-for-db! stub)
+ (sync-databases-trigger/check-and-schedule-tasks-for-db! stub)
(is (zero? @calls))))))))
(deftest check-orphaned-jobs-removed-test
diff --git a/test/metabase/warehouses_rest/api_test.clj b/test/metabase/warehouses_rest/api_test.clj
index 8c2ab511c19..856601110d4 100644
--- a/test/metabase/warehouses_rest/api_test.clj
+++ b/test/metabase/warehouses_rest/api_test.clj
@@ -30,6 +30,7 @@
[metabase.sync.sync-metadata :as sync-metadata]
[metabase.sync.task.sync-databases :as task.sync-databases]
[metabase.sync.task.sync-databases-test :as task.sync-databases-test]
+ [metabase.sync.task.sync-databases-trigger :as sync-databases-trigger]
[metabase.task.core :as task]
[metabase.test :as mt]
[metabase.test.data.impl :as data.impl]
@@ -373,7 +374,7 @@
(defn- sync-and-analyze-trigger-name
[db]
- (.getName ^TriggerKey (#'task.sync-databases/trigger-key db @#'task.sync-databases/sync-analyze-task-info)))
+ (.getName ^TriggerKey (#'sync-databases-trigger/trigger-key db sync-databases-trigger/sync-analyze-task-info)))
(defmacro with-test-driver-available!
[& body]