Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Recreate model index quartz triggers on job initialization when needed (#65394)

Noah Moss01f7eeb023ef
diff --git a/src/metabase/indexed_entities/task/index_values.clj b/src/metabase/indexed_entities/task/index_values.clj
index fe8fee7d4fa..20552fdda47 100644
--- a/src/metabase/indexed_entities/task/index_values.clj
+++ b/src/metabase/indexed_entities/task/index_values.clj
@@ -18,8 +18,9 @@
(set! *warn-on-reflection* true)
#_{:clj-kondo/ignore [:unused-private-var]}
-(def ^:private refreshable-states
- "States of a model index that are refreshable."
+;; Possible values for the :state field on model index records.
+;; Unused, but kept here for reference.
+(def ^:private model-index-states
#{"indexed" "initial" "error" "overflow"})
(defn- should-deindex?
@@ -99,9 +100,36 @@
(let [trigger-key (model-index-trigger-key (:id model-index))]
(task/delete-trigger! trigger-key)))
+(defn- recreate-missing-triggers!
+ "Ensure all model indexes in the database have triggers in Quartz."
+ []
+ (try
+ (let [existing-trigger-model-index-ids (try
+ (->> (task/job-info refresh-model-index-key)
+ :triggers
+ (keep #(get-in % [:data "model-index-id"]))
+ set)
+ (catch Exception e
+ (log/warn e "Error fetching existing triggers from Quartz, will recreate all triggers")
+ #{}))
+ missing-trigger-model-indexes (if (seq existing-trigger-model-index-ids)
+ (t2/select :model/ModelIndex :id [:not-in existing-trigger-model-index-ids])
+ (t2/select :model/ModelIndex))]
+ (when (seq missing-trigger-model-indexes)
+ (log/infof "Found %d model index(es) without triggers, recreating..."
+ (count missing-trigger-model-indexes))
+ (doseq [model-index missing-trigger-model-indexes]
+ (try
+ (add-indexing-job model-index)
+ (catch Exception e
+ (log/errorf e "Error re-adding indexing job for model-index: %d" (:id model-index)))))))
+ (catch Exception e
+ (log/error e "Error during model index trigger recreation"))))
+
(defn- job-init!
[]
- (task/add-job! refresh-job))
+ (task/add-job! refresh-job)
+ (recreate-missing-triggers!))
(defmethod task/init! ::ModelIndexValues
[_]
diff --git a/test/metabase/indexed_entities/models/model_index_test.clj b/test/metabase/indexed_entities/models/model_index_test.clj
index 78909ad758b..4cde76eee41 100644
--- a/test/metabase/indexed_entities/models/model_index_test.clj
+++ b/test/metabase/indexed_entities/models/model_index_test.clj
@@ -17,7 +17,7 @@
[metabase.util.malli.registry :as mr]
[toucan2.core :as t2]))
-(defmacro ^:private with-scheduler-setup! [& body]
+(defmacro with-scheduler-setup! [& body]
`(let [scheduler# (#'tu/in-memory-scheduler)]
;; need cross thread rebinding from with-redefs not a binding
(with-redefs [task/scheduler (constantly scheduler#)]
diff --git a/test/metabase/indexed_entities/task/index_values_test.clj b/test/metabase/indexed_entities/task/index_values_test.clj
new file mode 100644
index 00000000000..83b5a56ab72
--- /dev/null
+++ b/test/metabase/indexed_entities/task/index_values_test.clj
@@ -0,0 +1,47 @@
+(ns metabase.indexed-entities.task.index-values-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.indexed-entities.models.model-index-test :refer [with-scheduler-setup!]]
+ [metabase.indexed-entities.task.index-values :as task.index-values]
+ [metabase.task.impl :as task]
+ [metabase.test :as mt]
+ [metabase.util :as u]))
+
+(deftest recreate-missing-triggers-test
+ (with-scheduler-setup!
+ (mt/dataset test-data
+ (testing "Missing triggers are recreated on job-init"
+ (let [query (mt/mbql-query products)
+ pk-ref (mt/$ids $products.id)
+ value-ref (mt/$ids $products.title)
+ by-key (fn [k xs]
+ (some (fn [x] (when (= (:key x) k) x)) xs))]
+ (mt/with-temp [:model/Card model (assoc (mt/card-with-source-metadata-for-query query)
+ :type :model
+ :name "model index recreate trigger test")
+ :model/ModelIndex model-index {:model_id (u/the-id model)
+ :pk_ref pk-ref
+ :value_ref value-ref
+ :creator_id (mt/user->id :rasta)
+ :schedule "0 0 23 * * ? *"
+ :state "initial"}]
+ (let [trigger-key (format "metabase.task.IndexValues.trigger.%d" (:id model-index))
+ get-trigger #(->> (task/scheduler-info)
+ :jobs
+ (by-key "metabase.task.IndexValues.job")
+ :triggers
+ (by-key trigger-key))]
+ ;; create trigger by calling add-indexing-job
+ (task.index-values/add-indexing-job model-index)
+ (is (some? (get-trigger)) "Trigger should exist after creating model index")
+ ;; delete the trigger from Quartz
+ (task.index-values/remove-indexing-job model-index)
+ (is (nil? (get-trigger)) "Trigger should be deleted")
+ ;; call job-init! to recreate missing triggers
+ (#'task.index-values/job-init!)
+ (let [recreated-trigger (get-trigger)]
+ (is (some? recreated-trigger) "Trigger should be recreated by job-init!")
+ (is (= (:schedule model-index) (:schedule recreated-trigger))
+ "Recreated trigger should have correct schedule")
+ (is (= {"model-index-id" (:id model-index)} (:data recreated-trigger))
+ "Recreated trigger should have correct data")))))))))