Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

12.2 KB
Raw Blame
(ns metabase-enterprise.cache.task.refresh-cache-configs
  (:require
   [clojurewerkz.quartzite.jobs :as jobs]
   [clojurewerkz.quartzite.schedule.cron :as cron]
   [clojurewerkz.quartzite.triggers :as triggers]
   [java-time.api :as t]
   [metabase-enterprise.cache.db :as cache.db]
   [metabase-enterprise.cache.strategies :as strategies]
   [metabase.premium-features.core :as premium-features :refer [defenterprise]]
   [metabase.query-processor.core :as qp]
   [metabase.task.core :as task]
   [metabase.util.log :as log]
   [toucan2.core :as t2])
  (:import
   (java.util.concurrent Callable ExecutorService SynchronousQueue ThreadPoolExecutor TimeUnit)
   (org.apache.commons.lang3.concurrent BasicThreadFactory)
   (org.quartz.spi MutableTrigger)))

(set! *warn-on-reflection* true)

;;; ------------------------------------------- Preemptive Caching ----------------------------------------------------

(defonce ^:private pool
  (delay
    (ThreadPoolExecutor.
     0                     ;; core pool size
     10                    ;; max pool size (upper limit)
     100 TimeUnit/SECONDS  ;; keep-alive time for idle threads
     (SynchronousQueue.)   ;; direct handoff
     (.build
      (doto (BasicThreadFactory/builder)
        (.namingPattern "preemptive-caching-thread-pool-%d"))))))

#_{:clj-kondo/ignore [:metabase/discourage-dynamic-vars]}
(def ^:dynamic *parameterized-queries-to-rerun-per-card*
  "Number of query variations (e.g. with different parameters) to run for a single cached card."
  10)

#_{:clj-kondo/ignore [:metabase/discourage-dynamic-vars]}
(def ^:dynamic *run-cache-refresh-async*
  "Should cache refresh jobs be run asynchronously? Defaults to true, can be set to false for testing."
  true)

(defn- submit-refresh-task-async!
  "Submits a job to the thread pool to run a sequence of queries for a single card or dashboard being refreshed.
  This is best-effort; we try each query once and discard failures."
  [refresh-task-fn]
  (.submit ^ExecutorService @pool ^Callable refresh-task-fn))

(defn discarding-rff
  "Returns a reducing function that discards result rows"
  [metadata]
  (fn discarding-rf
    ([] {:data metadata})
    ([result] result)
    ([result _row] result)))

(defn- refresh-task
  "Returns a function that serially reruns queries based on `refresh-defs`, and discards the results. Each refresh
  definition contains a card-id, an optional dashboard-id, and a list of queries to rerun."
  [refresh-defs]
  (fn []
    (let [card-ids      (into #{} (map :card-id refresh-defs))
          cards-by-id   (cache.db/cards-by-id card-ids)
          all-db-ids    (into #{} (keep (comp :database_id val)) cards-by-id)
          router-db-ids (if (seq all-db-ids)
                          (cache.db/router-database-ids all-db-ids)
                          #{})]
      (doseq [{:keys [card-id dashboard-id queries]} refresh-defs]
        (let [card (get cards-by-id card-id)]
          ;; Skip cards on router databases — cache refresh runs without a user context, so database
          ;; routing will always fail for these cards (anonymous access is not allowed).
          (when-not (contains? router-db-ids (:database_id card))
            ;; Annotate the query with its cache strategy in the format expected by the QP
            (let [cache-strategy (strategies/cache-strategy card dashboard-id)]
              (doseq [query queries]
                (try
                  (qp/process-query
                   (qp/userland-query
                    (-> query
                        (assoc-in [:middleware :ignore-cached-results?] true)
                        (assoc :cache-strategy cache-strategy))
                    {:executed-by  nil
                     :context      :cache-refresh
                     :card-id      card-id
                     :dashboard-id dashboard-id})
                   discarding-rff)
                  (catch Exception e
                    (log/debugf "Error refreshing cache for card %s: %s" card-id (ex-message e))))))))))))

(defn- duration-ago
  [{:keys [duration unit]}]
  (t/minus (t/offset-date-time)
           (t/duration duration (keyword unit))))

(defn- select-parameterized-queries
  "Given a list of parameterized query definitions from the Query table with additional :count and :card-id keys,
  selects the 10 most common queries for each card ID that we should rerun."
  [parameterized-queries]
  (apply concat
         (-> (group-by :card-id parameterized-queries)
             (update-vals
              (fn [queries]
                (->> queries
                     (sort-by :count >)
                     (take *parameterized-queries-to-rerun-per-card*))))
             vals)))

(defn- started-after
  "Cutoff of the safety check that keeps us from scanning all of query_execution: a query not executed at all in the
  last month (including cache hits) is not refreshed again."
  []
  (duration-ago {:duration 30 :unit "days"}))

(defn- duration-scopes
  "The `{:model :model-id :rerun-cutoff}` scopes of the duration `cache-configs`."
  [cache-configs]
  (for [{:keys [model model_id config]} cache-configs]
    {:model model, :model-id model_id, :rerun-cutoff (duration-ago config)}))

(defn- duration-queries-to-rerun-for
  "The query definitions to rerun for the duration `cache-configs`; `parameterized?` selects parameterized or plain
  queries."
  [cache-configs parameterized?]
  (cache.db/duration-queries-to-rerun (duration-scopes cache-configs) (started-after) parameterized?))

(defn- duration-queries-to-rerun
  []
  (let [cache-configs (cache.db/duration-cache-configs)]
    (when (seq cache-configs)
      (let [base-queries          (duration-queries-to-rerun-for cache-configs false)
            parameterized-queries (duration-queries-to-rerun-for cache-configs true)]
        (concat base-queries (select-parameterized-queries parameterized-queries))))))

(defn- clear-caches-for-queries!
  "Deletes any existing cache entries for queries that we are about to re-run, so that subsequent tasks don't also try
  to re-run them before the cache has been refreshed. "
  [queries]
  (doseq [batch (partition 1000 1000 nil queries)]
    (cache.db/delete-query-caches! (map :cache-hash batch))))

(defn- maybe-refresh-duration-caches!
  "Detects caches with strategy=duration that are eligible for refreshing, and returns a count of the refresh jobs that
  were generated (i.e. the number of different cards refreshed, with each card potentially having multiple queries)."
  []
  (if-let [queries (seq (duration-queries-to-rerun))]
    (let [refresh-defs (->> queries
                            (group-by (juxt :card-id :dashboard-id))
                            (map (fn [[[card-id dashboard-id] queries]]
                                   {:card-id card-id
                                    :dashboard-id dashboard-id
                                    :queries (map :query queries)})))
          task         (refresh-task refresh-defs)]
      (clear-caches-for-queries! queries)
      (if *run-cache-refresh-async*
        (submit-refresh-task-async! task)
        (task))
      (count refresh-defs))
    0))

(defn- scheduled-base-query-to-rerun
  "The unparameterized query definition of the Card with `card-id` to rerun, or nil."
  [card-id]
  (cache.db/scheduled-base-query-to-rerun card-id (started-after)))

(defn- scheduled-parameterized-queries-to-rerun
  "The parameterized query definitions of the Card with `card-id` executed after `rerun-cutoff` to rerun."
  [card-id rerun-cutoff]
  (cache.db/scheduled-parameterized-queries-to-rerun card-id
                                                     rerun-cutoff
                                                     *parameterized-queries-to-rerun-per-card*))

(defn- scheduled-queries-to-rerun
  "Returns a list containing all of the parameterized query definitions that we should preemptively rerun for a given
  card that uses :schedule caching."
  [card-id rerun-cutoff]
  (let [base-query            (scheduled-base-query-to-rerun card-id)
        parameterized-queries (scheduled-parameterized-queries-to-rerun card-id rerun-cutoff)]
    (->> (concat (when base-query [base-query])
                 parameterized-queries)
         (map :query)
         distinct)))

(defn- schedule-cache-config->card-ids
  "Takes a schedule cache config and returns a sequence of card IDs to rerun."
  [{:keys [model_id model]}]
  (case model
    "question" [model_id]
    "dashboard" (let [dashboard (-> (cache.db/dashboard model_id)
                                    (t2/hydrate :dashcards))]
                  (distinct (keep :card_id (:dashcards dashboard))))))

(defn- refresh-schedule-cache!
  "Given a cache config with the :schedule strategy, preemptively rerun the query (and a fixed number of parameterized
  variants) so that fresh results are cached."
  [{model          :model
    model-id       :model_id
    strategy       :strategy
    invalidated-at :invalidated_at
    created-at     :created_at
    :as            cache-config}]
  (assert (= strategy :schedule))
  (let [rerun-cutoff (or invalidated-at created-at)
        card-ids     (schedule-cache-config->card-ids cache-config)
        dashboard-id (when (= model "dashboard") model-id)
        refresh-defs (distinct
                      (map
                       (fn [card-id]
                         {:dashboard-id dashboard-id
                          :card-id      card-id
                          :queries      (scheduled-queries-to-rerun card-id rerun-cutoff)})
                       card-ids))
        task         (refresh-task refresh-defs)]
    (if *run-cache-refresh-async*
      (submit-refresh-task-async! task)
      (task))))

;;; ------------------------------------------- Cache invalidation task ------------------------------------------------

(defn- select-ready-to-run
  "Fetch whatever cache configs for a given `strategy` are ready to be updated."
  [strategy]
  (cache.db/cache-configs-ready-to-run strategy (t/offset-date-time)))

(defn- calc-next-run
  "Calculate when a next run should happen based on a cron schedule"
  [^String schedule now]
  (let [^MutableTrigger cron (cron/finalize (cron/cron-schedule schedule))]
    ;; Synchronize cron runner to our app time, which may be mocked in tests
    (.setStartTime cron (t/java-date))
    (-> (.getFireTimeAfter cron (t/java-date now))
        (t/offset-date-time (t/zone-offset)))))

(defenterprise refresh-cache-configs!
  "Update `invalidated_at` for every cache config with `:schedule` strategy, and maybe rerun cached queries
  for both `:schedule` and `:duration` caches if preemptive caching is enabled."
  :feature :none
  []
  (let [now (t/offset-date-time)
        schedule-caches-to-invalidate (select-ready-to-run :schedule)
        schedule-refresh-count
        (reduce
         (fn [refreshed-count {:keys [id config refresh_automatically] :as cache-config}]
           (cache.db/update-cache-config! id {:next_run_at    (calc-next-run (:schedule config) now)
                                              :invalidated_at now})
           (if (and (premium-features/enable-preemptive-caching?) refresh_automatically)
             (do
               (refresh-schedule-cache! cache-config)
               (inc refreshed-count))
             refreshed-count))
         0
         schedule-caches-to-invalidate)
        duration-refresh-count
        (if (premium-features/enable-preemptive-caching?)
          (maybe-refresh-duration-caches!)
          0)]
    {:schedule-invalidated (count schedule-caches-to-invalidate)
     :schedule-refreshed   schedule-refresh-count
     :duration-refreshed   duration-refresh-count}))

(task/defjob Cache
  "Refresh 'schedule' caches"
  {:saved-class "metabase_enterprise.cache.task.refresh_cache_configs.Cache"
   :concurrent? false}
  [_ctx]
  (refresh-cache-configs!))

(def ^:private cache-job
  (jobs/build
   (jobs/with-description "Schedule Caches refresh task")
   (jobs/of-type Cache)
   (jobs/with-identity (jobs/key "metabase-enterprise.cache.job"))
   (jobs/store-durably)))

(def ^:private cache-trigger
  (triggers/build
   (triggers/with-identity (triggers/key "metabase-enterprise.cache.trigger"))
   (triggers/start-now)
   (triggers/with-schedule
    (cron/cron-schedule "0 * * * * ? *"))))

(defmethod task/init! ::RefreshCacheConfigs [_]
  (when (premium-features/has-feature? :cache-granular-controls)
    (task/schedule-task! cache-job cache-trigger)))