The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Health check per engine (#78813)
Alexander Polyankinae68432f3089
diff --git a/src/metabase/warehouses/models/database.clj b/src/metabase/warehouses/models/database.cljindex 30ae4aa8b2e..07dd6c80c64 100644--- a/src/metabase/warehouses/models/database.clj+++ b/src/metabase/warehouses/models/database.clj@@ -330,11 +330,35 @@(driver.conn/effective-details database)))](check-connection! database driver engine-str (assoc admin-details :engine engine-str) "admin"))))))))+(defn- health-check-candidates+ "The databases to health check at startup: one representative per unique engine — the lowest-id regular database of+ each engine — rather than every database. The health signal feeds the per-driver `:metabase-database/status`+ metrics, so checking hundreds of same-engine databases at startup mostly re-measures the same driver stack while+ hammering the warehouses with connection attempts. Audit/sample databases are excluded here (rather than relying on+ the guard in [[health-check-database!]]) so they can never claim an engine's representative slot and starve a real+ database of its check; router destinations are excluded because they can be very numerous and are only reachable+ through their router.++ The representative ids are aggregated in the database first, and only those rows are fetched as model instances:+ instances with 3k+ databases exist in the wild, and realizing every Database row (including details decryption)+ just to pick one per engine would defeat the point."+ []+ (let [ids (map :id (t2/query {:select [[:%min.id :id]]+ :from [(t2/table-name :model/Database)]+ :where [:and+ [:= :is_audit false]+ [:= :is_sample false]+ [:= :router_database_id nil]]+ :group-by [:engine]}))]+ (when (seq ids)+ (t2/select :model/Database :id [:in ids]))))+(defn check-health!- "Health checks databases connected to metabase asynchronously using a thread pool."+ "Health checks databases connected to metabase asynchronously using a thread pool. Only one database per unique+ engine is checked -- see [[health-check-candidates]]."[](analytics/clear! :metabase-database/status)- (doseq [database (t2/select :model/Database)]+ (doseq [database (health-check-candidates)](health-check-database! database)));; TODO - something like NSNotificationCenter in Objective-C would be really really useful here so things that want todiff --git a/test/metabase/warehouses/models/database_test.clj b/test/metabase/warehouses/models/database_test.cljindex 450e1cd9261..7d8d61c12e4 100644--- a/test/metabase/warehouses/models/database_test.clj+++ b/test/metabase/warehouses/models/database_test.clj@@ -89,12 +89,41 @@(is (= nil(trigger-for-db db-id))))))))+(deftest health-check-candidates-test+ (testing "startup health checks pick one representative database per engine: the lowest id, skipping+ audit/sample/destination databases"+ (mt/with-temp [:model/Database {router :id} {:engine :postgres}+ :model/Database {dest :id} {:engine :mysql :router_database_id router}+ :model/Database {sample :id} {:engine :mysql :is_sample true}+ :model/Database {audit :id} {:engine :mysql :is_audit true}+ :model/Database {_mysql :id} {:engine :mysql}+ :model/Database {pg2 :id} {:engine :postgres}+ :model/Database {pg3 :id} {:engine :postgres}]+ (let [candidates (#'database/health-check-candidates)+ candidate-ids (into #{} (map :id) candidates)+ engines (into #{} (map :engine) candidates)]+ (testing "exactly one representative per engine"+ (is (= (count candidates) (count engines))))+ (testing "every engine with an eligible database gets a representative"+ (is (contains? engines :postgres))+ (is (contains? engines :mysql)))+ (testing "only the lowest-id database of an engine can be the representative"+ (is (not (contains? candidate-ids pg2)))+ (is (not (contains? candidate-ids pg3))))+ (testing "router destinations are never candidates"+ (is (not (contains? candidate-ids dest))))+ (testing "sample databases can't claim an engine's slot"+ (is (not (contains? candidate-ids sample))))+ (testing "audit databases are never candidates"+ (is (not (contains? candidate-ids audit))))))))+(deftest check-health!-test(mt/test-drivers (mt/normal-drivers)(let [original-select (mt/original-fn #'t2/select)](mt/with-dynamic-fn-redefs [quick-task/submit-task! (fn [task] (task))+ ;; make the startup candidate fetch return only the test DBt2/select (fn [model & args]- (if (and (= model :model/Database) (empty? args))+ (if (= model :model/Database)[(mt/db)](apply original-select model args)))](binding [driver.settings/*allow-testing-h2-connections* true]