Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Health check per engine (#78813)

Alexander Polyankinae68432f3089
diff --git a/src/metabase/warehouses/models/database.clj b/src/metabase/warehouses/models/database.clj
index 30ae4aa8b2e..07dd6c80c64 100644
--- a/src/metabase/warehouses/models/database.clj
+++ b/src/metabase/warehouses/models/database.clj
@@ -330,11 +330,35 @@
(driver.conn/effective-details database)))]
(check-connection! database driver engine-str (assoc admin-details :engine engine-str) "admin"))))))))
+(defn- health-check-candidates
+ "The databases to health check at startup: one representative per unique engine — the lowest-id regular database of
+ each engine — rather than every database. The health signal feeds the per-driver `:metabase-database/status`
+ metrics, so checking hundreds of same-engine databases at startup mostly re-measures the same driver stack while
+ hammering the warehouses with connection attempts. Audit/sample databases are excluded here (rather than relying on
+ the guard in [[health-check-database!]]) so they can never claim an engine's representative slot and starve a real
+ database of its check; router destinations are excluded because they can be very numerous and are only reachable
+ through their router.
+
+ The representative ids are aggregated in the database first, and only those rows are fetched as model instances:
+ instances with 3k+ databases exist in the wild, and realizing every Database row (including details decryption)
+ just to pick one per engine would defeat the point."
+ []
+ (let [ids (map :id (t2/query {:select [[:%min.id :id]]
+ :from [(t2/table-name :model/Database)]
+ :where [:and
+ [:= :is_audit false]
+ [:= :is_sample false]
+ [:= :router_database_id nil]]
+ :group-by [:engine]}))]
+ (when (seq ids)
+ (t2/select :model/Database :id [:in ids]))))
+
(defn check-health!
- "Health checks databases connected to metabase asynchronously using a thread pool."
+ "Health checks databases connected to metabase asynchronously using a thread pool. Only one database per unique
+ engine is checked -- see [[health-check-candidates]]."
[]
(analytics/clear! :metabase-database/status)
- (doseq [database (t2/select :model/Database)]
+ (doseq [database (health-check-candidates)]
(health-check-database! database)))
;; TODO - something like NSNotificationCenter in Objective-C would be really really useful here so things that want to
diff --git a/test/metabase/warehouses/models/database_test.clj b/test/metabase/warehouses/models/database_test.clj
index 450e1cd9261..7d8d61c12e4 100644
--- a/test/metabase/warehouses/models/database_test.clj
+++ b/test/metabase/warehouses/models/database_test.clj
@@ -89,12 +89,41 @@
(is (= nil
(trigger-for-db db-id))))))))
+(deftest health-check-candidates-test
+ (testing "startup health checks pick one representative database per engine: the lowest id, skipping
+ audit/sample/destination databases"
+ (mt/with-temp [:model/Database {router :id} {:engine :postgres}
+ :model/Database {dest :id} {:engine :mysql :router_database_id router}
+ :model/Database {sample :id} {:engine :mysql :is_sample true}
+ :model/Database {audit :id} {:engine :mysql :is_audit true}
+ :model/Database {_mysql :id} {:engine :mysql}
+ :model/Database {pg2 :id} {:engine :postgres}
+ :model/Database {pg3 :id} {:engine :postgres}]
+ (let [candidates (#'database/health-check-candidates)
+ candidate-ids (into #{} (map :id) candidates)
+ engines (into #{} (map :engine) candidates)]
+ (testing "exactly one representative per engine"
+ (is (= (count candidates) (count engines))))
+ (testing "every engine with an eligible database gets a representative"
+ (is (contains? engines :postgres))
+ (is (contains? engines :mysql)))
+ (testing "only the lowest-id database of an engine can be the representative"
+ (is (not (contains? candidate-ids pg2)))
+ (is (not (contains? candidate-ids pg3))))
+ (testing "router destinations are never candidates"
+ (is (not (contains? candidate-ids dest))))
+ (testing "sample databases can't claim an engine's slot"
+ (is (not (contains? candidate-ids sample))))
+ (testing "audit databases are never candidates"
+ (is (not (contains? candidate-ids audit))))))))
+
(deftest check-health!-test
(mt/test-drivers (mt/normal-drivers)
(let [original-select (mt/original-fn #'t2/select)]
(mt/with-dynamic-fn-redefs [quick-task/submit-task! (fn [task] (task))
+ ;; make the startup candidate fetch return only the test DB
t2/select (fn [model & args]
- (if (and (= model :model/Database) (empty? args))
+ (if (= model :model/Database)
[(mt/db)]
(apply original-select model args)))]
(binding [driver.settings/*allow-testing-h2-connections* true]