The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
(ns metabase-enterprise.cache.task.refresh-cache-configs
(:require
[clojurewerkz.quartzite.jobs :as jobs]
[clojurewerkz.quartzite.schedule.cron :as cron]
[clojurewerkz.quartzite.triggers :as triggers]
[java-time.api :as t]
[metabase-enterprise.cache.db :as cache.db]
[metabase-enterprise.cache.strategies :as strategies]
[metabase.premium-features.core :as premium-features :refer [defenterprise]]
[metabase.query-processor.core :as qp]
[metabase.task.core :as task]
[metabase.util.log :as log]
[toucan2.core :as t2])
(:import
(java.util.concurrent Callable ExecutorService SynchronousQueue ThreadPoolExecutor TimeUnit)
(org.apache.commons.lang3.concurrent BasicThreadFactory)
(org.quartz.spi MutableTrigger)))
(set! *warn-on-reflection* true)
;;; ------------------------------------------- Preemptive Caching ----------------------------------------------------
(defonce ^:private pool
(delay
(ThreadPoolExecutor.
0 ;; core pool size
10 ;; max pool size (upper limit)
100 TimeUnit/SECONDS ;; keep-alive time for idle threads
(SynchronousQueue.) ;; direct handoff
(.build
(doto (BasicThreadFactory/builder)
(.namingPattern "preemptive-caching-thread-pool-%d"))))))
#_{:clj-kondo/ignore [:metabase/discourage-dynamic-vars]}
(def ^:dynamic *parameterized-queries-to-rerun-per-card*
"Number of query variations (e.g. with different parameters) to run for a single cached card."
10)
#_{:clj-kondo/ignore [:metabase/discourage-dynamic-vars]}
(def ^:dynamic *run-cache-refresh-async*
"Should cache refresh jobs be run asynchronously? Defaults to true, can be set to false for testing."
true)
(defn- submit-refresh-task-async!
"Submits a job to the thread pool to run a sequence of queries for a single card or dashboard being refreshed.
This is best-effort; we try each query once and discard failures."
[refresh-task-fn]
(.submit ^ExecutorService @pool ^Callable refresh-task-fn))
(defn discarding-rff
"Returns a reducing function that discards result rows"
[metadata]
(fn discarding-rf
([] {:data metadata})
([result] result)
([result _row] result)))
(defn- refresh-task
"Returns a function that serially reruns queries based on `refresh-defs`, and discards the results. Each refresh
definition contains a card-id, an optional dashboard-id, and a list of queries to rerun."
[refresh-defs]
(fn []
(let [card-ids (into #{} (map :card-id refresh-defs))
cards-by-id (cache.db/cards-by-id card-ids)
all-db-ids (into #{} (keep (comp :database_id val)) cards-by-id)
router-db-ids (if (seq all-db-ids)
(cache.db/router-database-ids all-db-ids)
#{})]
(doseq [{:keys [card-id dashboard-id queries]} refresh-defs]
(let [card (get cards-by-id card-id)]
;; Skip cards on router databases — cache refresh runs without a user context, so database
;; routing will always fail for these cards (anonymous access is not allowed).
(when-not (contains? router-db-ids (:database_id card))
;; Annotate the query with its cache strategy in the format expected by the QP
(let [cache-strategy (strategies/cache-strategy card dashboard-id)]
(doseq [query queries]
(try
(qp/process-query
(qp/userland-query
(-> query
(assoc-in [:middleware :ignore-cached-results?] true)
(assoc :cache-strategy cache-strategy))
{:executed-by nil
:context :cache-refresh
:card-id card-id
:dashboard-id dashboard-id})
discarding-rff)
(catch Exception e
(log/debugf "Error refreshing cache for card %s: %s" card-id (ex-message e))))))))))))
(defn- duration-ago
[{:keys [duration unit]}]
(t/minus (t/offset-date-time)
(t/duration duration (keyword unit))))
(defn- select-parameterized-queries
"Given a list of parameterized query definitions from the Query table with additional :count and :card-id keys,
selects the 10 most common queries for each card ID that we should rerun."
[parameterized-queries]
(apply concat
(-> (group-by :card-id parameterized-queries)
(update-vals
(fn [queries]
(->> queries
(sort-by :count >)
(take *parameterized-queries-to-rerun-per-card*))))
vals)))
(defn- started-after
"Cutoff of the safety check that keeps us from scanning all of query_execution: a query not executed at all in the
last month (including cache hits) is not refreshed again."
[]
(duration-ago {:duration 30 :unit "days"}))
(defn- duration-scopes
"The `{:model :model-id :rerun-cutoff}` scopes of the duration `cache-configs`."
[cache-configs]
(for [{:keys [model model_id config]} cache-configs]
{:model model, :model-id model_id, :rerun-cutoff (duration-ago config)}))
(defn- duration-queries-to-rerun-for
"The query definitions to rerun for the duration `cache-configs`; `parameterized?` selects parameterized or plain
queries."
[cache-configs parameterized?]
(cache.db/duration-queries-to-rerun (duration-scopes cache-configs) (started-after) parameterized?))
(defn- duration-queries-to-rerun
[]
(let [cache-configs (cache.db/duration-cache-configs)]
(when (seq cache-configs)
(let [base-queries (duration-queries-to-rerun-for cache-configs false)
parameterized-queries (duration-queries-to-rerun-for cache-configs true)]
(concat base-queries (select-parameterized-queries parameterized-queries))))))
(defn- clear-caches-for-queries!
"Deletes any existing cache entries for queries that we are about to re-run, so that subsequent tasks don't also try
to re-run them before the cache has been refreshed. "
[queries]
(doseq [batch (partition 1000 1000 nil queries)]
(cache.db/delete-query-caches! (map :cache-hash batch))))
(defn- maybe-refresh-duration-caches!
"Detects caches with strategy=duration that are eligible for refreshing, and returns a count of the refresh jobs that
were generated (i.e. the number of different cards refreshed, with each card potentially having multiple queries)."
[]
(if-let [queries (seq (duration-queries-to-rerun))]
(let [refresh-defs (->> queries
(group-by (juxt :card-id :dashboard-id))
(map (fn [[[card-id dashboard-id] queries]]
{:card-id card-id
:dashboard-id dashboard-id
:queries (map :query queries)})))
task (refresh-task refresh-defs)]
(clear-caches-for-queries! queries)
(if *run-cache-refresh-async*
(submit-refresh-task-async! task)
(task))
(count refresh-defs))
0))
(defn- scheduled-base-query-to-rerun
"The unparameterized query definition of the Card with `card-id` to rerun, or nil."
[card-id]
(cache.db/scheduled-base-query-to-rerun card-id (started-after)))
(defn- scheduled-parameterized-queries-to-rerun
"The parameterized query definitions of the Card with `card-id` executed after `rerun-cutoff` to rerun."
[card-id rerun-cutoff]
(cache.db/scheduled-parameterized-queries-to-rerun card-id
rerun-cutoff
*parameterized-queries-to-rerun-per-card*))
(defn- scheduled-queries-to-rerun
"Returns a list containing all of the parameterized query definitions that we should preemptively rerun for a given
card that uses :schedule caching."
[card-id rerun-cutoff]
(let [base-query (scheduled-base-query-to-rerun card-id)
parameterized-queries (scheduled-parameterized-queries-to-rerun card-id rerun-cutoff)]
(->> (concat (when base-query [base-query])
parameterized-queries)
(map :query)
distinct)))
(defn- schedule-cache-config->card-ids
"Takes a schedule cache config and returns a sequence of card IDs to rerun."
[{:keys [model_id model]}]
(case model
"question" [model_id]
"dashboard" (let [dashboard (-> (cache.db/dashboard model_id)
(t2/hydrate :dashcards))]
(distinct (keep :card_id (:dashcards dashboard))))))
(defn- refresh-schedule-cache!
"Given a cache config with the :schedule strategy, preemptively rerun the query (and a fixed number of parameterized
variants) so that fresh results are cached."
[{model :model
model-id :model_id
strategy :strategy
invalidated-at :invalidated_at
created-at :created_at
:as cache-config}]
(assert (= strategy :schedule))
(let [rerun-cutoff (or invalidated-at created-at)
card-ids (schedule-cache-config->card-ids cache-config)
dashboard-id (when (= model "dashboard") model-id)
refresh-defs (distinct
(map
(fn [card-id]
{:dashboard-id dashboard-id
:card-id card-id
:queries (scheduled-queries-to-rerun card-id rerun-cutoff)})
card-ids))
task (refresh-task refresh-defs)]
(if *run-cache-refresh-async*
(submit-refresh-task-async! task)
(task))))
;;; ------------------------------------------- Cache invalidation task ------------------------------------------------
(defn- select-ready-to-run
"Fetch whatever cache configs for a given `strategy` are ready to be updated."
[strategy]
(cache.db/cache-configs-ready-to-run strategy (t/offset-date-time)))
(defn- calc-next-run
"Calculate when a next run should happen based on a cron schedule"
[^String schedule now]
(let [^MutableTrigger cron (cron/finalize (cron/cron-schedule schedule))]
;; Synchronize cron runner to our app time, which may be mocked in tests
(.setStartTime cron (t/java-date))
(-> (.getFireTimeAfter cron (t/java-date now))
(t/offset-date-time (t/zone-offset)))))
(defenterprise refresh-cache-configs!
"Update `invalidated_at` for every cache config with `:schedule` strategy, and maybe rerun cached queries
for both `:schedule` and `:duration` caches if preemptive caching is enabled."
:feature :none
[]
(let [now (t/offset-date-time)
schedule-caches-to-invalidate (select-ready-to-run :schedule)
schedule-refresh-count
(reduce
(fn [refreshed-count {:keys [id config refresh_automatically] :as cache-config}]
(cache.db/update-cache-config! id {:next_run_at (calc-next-run (:schedule config) now)
:invalidated_at now})
(if (and (premium-features/enable-preemptive-caching?) refresh_automatically)
(do
(refresh-schedule-cache! cache-config)
(inc refreshed-count))
refreshed-count))
0
schedule-caches-to-invalidate)
duration-refresh-count
(if (premium-features/enable-preemptive-caching?)
(maybe-refresh-duration-caches!)
0)]
{:schedule-invalidated (count schedule-caches-to-invalidate)
:schedule-refreshed schedule-refresh-count
:duration-refreshed duration-refresh-count}))
(task/defjob Cache
"Refresh 'schedule' caches"
{:saved-class "metabase_enterprise.cache.task.refresh_cache_configs.Cache"
:concurrent? false}
[_ctx]
(refresh-cache-configs!))
(def ^:private cache-job
(jobs/build
(jobs/with-description "Schedule Caches refresh task")
(jobs/of-type Cache)
(jobs/with-identity (jobs/key "metabase-enterprise.cache.job"))
(jobs/store-durably)))
(def ^:private cache-trigger
(triggers/build
(triggers/with-identity (triggers/key "metabase-enterprise.cache.trigger"))
(triggers/start-now)
(triggers/with-schedule
(cron/cron-schedule "0 * * * * ? *"))))
(defmethod task/init! ::RefreshCacheConfigs [_]
(when (premium-features/has-feature? :cache-granular-controls)
(task/schedule-task! cache-job cache-trigger)))