The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Use an isolated executor pool for test isolation (#79913)
Ben Grabow6d4769584ef2
diff --git a/test/metabase/data_studio/api/table_test.clj b/test/metabase/data_studio/api/table_test.cljindex 30d6c2037e7..99079d65e41 100644--- a/test/metabase/data_studio/api/table_test.clj+++ b/test/metabase/data_studio/api/table_test.clj@@ -7,8 +7,9 @@[metabase.sync.core :as sync][metabase.test :as mt][metabase.test.fixtures :as fixtures]+ [metabase.util.quick-task :as quick-task][toucan2.core :as t2])- (:import (java.util.concurrent CountDownLatch TimeUnit)))+ (:import (java.util.concurrent CountDownLatch Executors TimeUnit)))(set! *warn-on-reflection* true)@@ -97,24 +98,34 @@;; lot more to test here but will wait for firmer ground(testing "Can we trigger a metadata sync for a filtered set of tables"(let [tables (atom [])- latch (CountDownLatch. 4)]- (mt/with-temp [:model/Database {d1 :id} {:engine "h2", :details (:details (mt/db))}- :model/Database {d2 :id} {:engine "h2", :details (:details (mt/db))}- :model/Table {t1 :id} {:db_id d1, :schema "PUBLIC"}- :model/Table {t2 :id} {:db_id d1, :schema "PUBLIC"}- :model/Table {_ :id} {:db_id d2, :schema "PUBLIC"}- :model/Table {t4 :id} {:db_id d2, :schema "PUBLIC"}- :model/Table {t5 :id} {:db_id d2, :schema "FOO"}]- (mt/with-dynamic-fn-redefs [sync/sync-table! (fn [table]- (swap! tables conj table)- (.countDown latch)- nil)]- (mt/user-http-request :crowberto :post 204 "data-studio/table/sync-schema" {:database_ids [d1],- :schema_ids [(format "%d:FOO" d2)]- :table_ids [t4]})- (testing "sync called?"- (is (true? (.await latch 4 TimeUnit/SECONDS)))- (is (= [t1 t2 t4 t5] (map :id @tables)))))))))+ latch (CountDownLatch. 4)+ ;; Run on a pool of our own: the shared one is process-wide and holds fire-and-forget tasks left+ ;; behind by earlier tests, each with the default two-hour timeout. One of those still running+ ;; ahead of these syncs delays them past the await below, which is what happens on driver CI,+ ;; where those leftover tasks are real syncs over the network. Keep it single-threaded: the+ ;; endpoint submits one task per table, and the assertion below reads their order.+ pool (Executors/newSingleThreadExecutor)]+ (try+ (mt/with-temp [:model/Database {d1 :id} {:engine "h2", :details (:details (mt/db))}+ :model/Database {d2 :id} {:engine "h2", :details (:details (mt/db))}+ :model/Table {t1 :id} {:db_id d1, :schema "PUBLIC"}+ :model/Table {t2 :id} {:db_id d1, :schema "PUBLIC"}+ :model/Table {_ :id} {:db_id d2, :schema "PUBLIC"}+ :model/Table {t4 :id} {:db_id d2, :schema "PUBLIC"}+ :model/Table {t5 :id} {:db_id d2, :schema "FOO"}]+ (with-redefs [quick-task/executor (delay pool)]+ (mt/with-dynamic-fn-redefs [sync/sync-table! (fn [table]+ (swap! tables conj table)+ (.countDown latch)+ nil)]+ (mt/user-http-request :crowberto :post 204 "data-studio/table/sync-schema" {:database_ids [d1],+ :schema_ids [(format "%d:FOO" d2)]+ :table_ids [t4]})+ (testing "sync called?"+ (is (true? (.await latch 4 TimeUnit/SECONDS)))+ (is (= [t1 t2 t4 t5] (map :id @tables)))))))+ (finally+ (.shutdownNow pool))))))(deftest ^:parallel non-admins-cant-trigger-bulk-rescan-values-test(testing "Non-admins should not be allowed to trigger rescan values"@@ -125,24 +136,31 @@;; lot more to test here but will wait for firmer ground(testing "Can we trigger a field values sync for a filtered set of tables"(let [tables (atom [])- latch (CountDownLatch. 4)]- (mt/with-temp [:model/Database {d1 :id} {:engine "h2", :details (:details (mt/db))}- :model/Database {d2 :id} {:engine "h2", :details (:details (mt/db))}- :model/Table {t1 :id} {:db_id d1, :schema "PUBLIC"}- :model/Table {t2 :id} {:db_id d1, :schema "PUBLIC"}- :model/Table {_ :id} {:db_id d2, :schema "PUBLIC"}- :model/Table {t4 :id} {:db_id d2, :schema "PUBLIC"}- :model/Table {t5 :id} {:db_id d2, :schema "FOO"}]- (mt/with-dynamic-fn-redefs [sync/update-field-values-for-table! (fn [table]- (swap! tables conj table)- (.countDown latch)- nil)]- (mt/user-http-request :crowberto :post 204 "data-studio/table/rescan-values" {:database_ids [d1],- :schema_ids [(format "%d:FOO" d2)]- :table_ids [t4]})- (testing "rescanned?"- (is (true? (.await latch 4 TimeUnit/SECONDS)))- (is (= [t1 t2 t4 t5] (map :id @tables)))))))))+ latch (CountDownLatch. 4)+ ;; Isolated single-thread pool, for the same reasons as in+ ;; `trigger-bulk-metadata-sync-for-table-test`.+ pool (Executors/newSingleThreadExecutor)]+ (try+ (mt/with-temp [:model/Database {d1 :id} {:engine "h2", :details (:details (mt/db))}+ :model/Database {d2 :id} {:engine "h2", :details (:details (mt/db))}+ :model/Table {t1 :id} {:db_id d1, :schema "PUBLIC"}+ :model/Table {t2 :id} {:db_id d1, :schema "PUBLIC"}+ :model/Table {_ :id} {:db_id d2, :schema "PUBLIC"}+ :model/Table {t4 :id} {:db_id d2, :schema "PUBLIC"}+ :model/Table {t5 :id} {:db_id d2, :schema "FOO"}]+ (with-redefs [quick-task/executor (delay pool)]+ (mt/with-dynamic-fn-redefs [sync/update-field-values-for-table! (fn [table]+ (swap! tables conj table)+ (.countDown latch)+ nil)]+ (mt/user-http-request :crowberto :post 204 "data-studio/table/rescan-values" {:database_ids [d1],+ :schema_ids [(format "%d:FOO" d2)]+ :table_ids [t4]})+ (testing "rescanned?"+ (is (true? (.await latch 4 TimeUnit/SECONDS)))+ (is (= [t1 t2 t4 t5] (map :id @tables)))))))+ (finally+ (.shutdownNow pool))))))(deftest ^:parallel non-admins-cant-trigger-bulk-discard-values-test(testing "Non-admins should not be allowed to trigger discard values"diff --git a/test/metabase/warehouse_schema_rest/api/table_test.clj b/test/metabase/warehouse_schema_rest/api/table_test.cljindex d5758b18148..a1b513b8ed7 100644--- a/test/metabase/warehouse_schema_rest/api/table_test.clj+++ b/test/metabase/warehouse_schema_rest/api/table_test.clj@@ -24,7 +24,9 @@[metabase.util :as u][metabase.util.quick-task :as quick-task][metabase.warehouse-schema-rest.api.table :as api.table]- [toucan2.core :as t2]))+ [toucan2.core :as t2])+ (:import+ (java.util.concurrent Executors)))(set! *warn-on-reflection* true)@@ -1290,18 +1292,27 @@(testing "POST /api/table/:id/sync_schema"(testing "User with manage-table-metadata permission can sync table"(let [sync-called? (promise)- timeout (* 60 1000)]- (mt/with-premium-features #{:audit-app}- (mt/with-temp [:model/Database {db-id :id} {:engine "h2", :details (:details (mt/db))}- :model/Table table {:db_id db-id :schema "PUBLIC"}]- (mt/with-no-data-perms-for-all-users!- ;; Grant only manage-table-metadata permission for this table- (data-perms/set-table-permission! (perms-group/all-users) (:id table) :perms/manage-table-metadata :yes)- (with-redefs [sync/sync-table! (deliver-when-tbl sync-called? table)]- (mt/user-http-request :rasta :post 200 (format "table/%d/sync_schema" (u/the-id table)))- (testing "sync called?"- (is (true?- (deref sync-called? timeout :sync-never-called))))))))))))+ timeout (* 60 1000)+ ;; Isolated pool: the shared one is process-wide and holds fire-and-forget tasks left behind by+ ;; earlier tests, each with the default two-hour timeout. One of those still running ahead of+ ;; this sync starves it past the deref below, which is what happens on driver CI, where those+ ;; leftover tasks are real syncs over the network.+ pool (Executors/newSingleThreadExecutor)]+ (try+ (mt/with-premium-features #{:audit-app}+ (mt/with-temp [:model/Database {db-id :id} {:engine "h2", :details (:details (mt/db))}+ :model/Table table {:db_id db-id :schema "PUBLIC"}]+ (mt/with-no-data-perms-for-all-users!+ ;; Grant only manage-table-metadata permission for this table+ (data-perms/set-table-permission! (perms-group/all-users) (:id table) :perms/manage-table-metadata :yes)+ (with-redefs [quick-task/executor (delay pool)]+ (mt/with-dynamic-fn-redefs [sync/sync-table! (deliver-when-tbl sync-called? table)]+ (mt/user-http-request :rasta :post 200 (format "table/%d/sync_schema" (u/the-id table)))+ (testing "sync called?"+ (is (true?+ (deref sync-called? timeout :sync-never-called)))))))))+ (finally+ (.shutdownNow pool)))))))(deftest sync-schema-mirror-database-test(testing "POST /api/table/:id/sync_schema"diff --git a/test/metabase/warehouses_rest/api_test.clj b/test/metabase/warehouses_rest/api_test.cljindex b9fafe4426b..1763e3b768f 100644--- a/test/metabase/warehouses_rest/api_test.clj+++ b/test/metabase/warehouses_rest/api_test.clj@@ -51,7 +51,7 @@[toucan2.core :as t2])(:import(java.sql Connection)- (java.util.concurrent CountDownLatch)+ (java.util.concurrent CountDownLatch Executors)(org.quartz JobDetail TriggerKey)))(set! *warn-on-reflection* true)@@ -1735,9 +1735,15 @@(deftest sync-schema-executes-when-executor-busy-test(testing "POST /api/database/:id/sync_schema should execute sync even when quick-task executor is busy (GHY-3254)"(let [sync-called? (promise)- blocker-latch (CountDownLatch. 1)]+ blocker-latch (CountDownLatch. 1)+ ;; Run on a pool of our own: the shared one is process-wide and holds fire-and-forget tasks left+ ;; behind by earlier tests, each with the default two-hour timeout. One of those still running+ ;; ahead of the blocker delays everything below it past the deref timeout, which is what happens+ ;; on driver CI, where those leftover tasks are real syncs over the network.+ pool (Executors/newSingleThreadExecutor)](mt/with-temp [:model/Database {db-id :id} {:engine "h2" :details (:details (mt/db))}]- (with-redefs [sync-metadata/sync-db-metadata-explicit! (deliver-when-db sync-called? db-id)+ (with-redefs [quick-task/executor (delay pool)+ sync-metadata/sync-db-metadata-explicit! (deliver-when-db sync-called? db-id)analyze/analyze-db-explicit! (constantly nil)];; Submit a blocking task with a 1-second timeout so it gets cancelled quickly.;; This simulates a stuck sync (e.g., hanging JDBC connection) that exceeds@@ -1751,7 +1757,8 @@(testing "sync executes after stuck task is evicted"(is (true? (deref sync-called? 10000 :sync-never-called))))(finally- (.countDown blocker-latch))))))))+ (.countDown blocker-latch)+ (.shutdownNow pool))))))))(deftest ^:parallel dismiss-spinner-test(testing "Can we dismiss the spinner? (#20863)"@@ -1778,22 +1785,28 @@(deftest can-rescan-fieldvalues-for-a-db(testing "Can we RESCAN all the FieldValues for a DB?"(mt/with-premium-features #{:audit-app}- (let [update-field-values-called? (promise)]- (mt/with-temp [:model/Database db {:engine "h2", :details (:details (mt/db))}]- (mt/with-dynamic-fn-redefs [sync.field-values/update-field-values! (fn [synced-db]- (when (= (u/the-id synced-db) (u/the-id db))- (deliver update-field-values-called? :sync-called)))]- (snowplow-test/with-fake-snowplow-collector- (mt/user-http-request :crowberto :post 200 (format "database/%d/rescan_values" (u/the-id db)))- (is (= :sync-called- (deref update-field-values-called? long-timeout :sync-never-called)))- (is (= (:id db) (:model_id (mt/latest-audit-log-entry "database-manual-scan"))))- (is (= (:id db) (-> (mt/latest-audit-log-entry "database-manual-scan")- :details :id)))- (testing "triggers snowplow event"- (is (=?- {"event" "database_manual_scan", "target_id" (u/the-id db)}- (:data (last (snowplow-test/pop-event-data-and-user-id!)))))))))))))+ (let [update-field-values-called? (promise)+ ;; Isolated pool, for the same reasons as in `sync-schema-executes-when-executor-busy-test`.+ pool (Executors/newSingleThreadExecutor)]+ (try+ (mt/with-temp [:model/Database db {:engine "h2", :details (:details (mt/db))}]+ (with-redefs [quick-task/executor (delay pool)]+ (mt/with-dynamic-fn-redefs [sync.field-values/update-field-values! (fn [synced-db]+ (when (= (u/the-id synced-db) (u/the-id db))+ (deliver update-field-values-called? :sync-called)))]+ (snowplow-test/with-fake-snowplow-collector+ (mt/user-http-request :crowberto :post 200 (format "database/%d/rescan_values" (u/the-id db)))+ (is (= :sync-called+ (deref update-field-values-called? long-timeout :sync-never-called)))+ (is (= (:id db) (:model_id (mt/latest-audit-log-entry "database-manual-scan"))))+ (is (= (:id db) (-> (mt/latest-audit-log-entry "database-manual-scan")+ :details :id)))+ (testing "triggers snowplow event"+ (is (=?+ {"event" "database_manual_scan", "target_id" (u/the-id db)}+ (:data (last (snowplow-test/pop-event-data-and-user-id!))))))))))+ (finally+ (.shutdownNow pool)))))))(deftest ^:parallel nonadmins-cant-trigger-rescan-test(testing "Non-admins should not be allowed to trigger re-scan"