Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Use an isolated executor pool for test isolation (#79913)

Ben Grabow6d4769584ef2
diff --git a/test/metabase/data_studio/api/table_test.clj b/test/metabase/data_studio/api/table_test.clj
index 30d6c2037e7..99079d65e41 100644
--- a/test/metabase/data_studio/api/table_test.clj
+++ b/test/metabase/data_studio/api/table_test.clj
@@ -7,8 +7,9 @@
[metabase.sync.core :as sync]
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
+ [metabase.util.quick-task :as quick-task]
[toucan2.core :as t2])
- (:import (java.util.concurrent CountDownLatch TimeUnit)))
+ (:import (java.util.concurrent CountDownLatch Executors TimeUnit)))
(set! *warn-on-reflection* true)
@@ -97,24 +98,34 @@
;; lot more to test here but will wait for firmer ground
(testing "Can we trigger a metadata sync for a filtered set of tables"
(let [tables (atom [])
- latch (CountDownLatch. 4)]
- (mt/with-temp [:model/Database {d1 :id} {:engine "h2", :details (:details (mt/db))}
- :model/Database {d2 :id} {:engine "h2", :details (:details (mt/db))}
- :model/Table {t1 :id} {:db_id d1, :schema "PUBLIC"}
- :model/Table {t2 :id} {:db_id d1, :schema "PUBLIC"}
- :model/Table {_ :id} {:db_id d2, :schema "PUBLIC"}
- :model/Table {t4 :id} {:db_id d2, :schema "PUBLIC"}
- :model/Table {t5 :id} {:db_id d2, :schema "FOO"}]
- (mt/with-dynamic-fn-redefs [sync/sync-table! (fn [table]
- (swap! tables conj table)
- (.countDown latch)
- nil)]
- (mt/user-http-request :crowberto :post 204 "data-studio/table/sync-schema" {:database_ids [d1],
- :schema_ids [(format "%d:FOO" d2)]
- :table_ids [t4]})
- (testing "sync called?"
- (is (true? (.await latch 4 TimeUnit/SECONDS)))
- (is (= [t1 t2 t4 t5] (map :id @tables)))))))))
+ latch (CountDownLatch. 4)
+ ;; Run on a pool of our own: the shared one is process-wide and holds fire-and-forget tasks left
+ ;; behind by earlier tests, each with the default two-hour timeout. One of those still running
+ ;; ahead of these syncs delays them past the await below, which is what happens on driver CI,
+ ;; where those leftover tasks are real syncs over the network. Keep it single-threaded: the
+ ;; endpoint submits one task per table, and the assertion below reads their order.
+ pool (Executors/newSingleThreadExecutor)]
+ (try
+ (mt/with-temp [:model/Database {d1 :id} {:engine "h2", :details (:details (mt/db))}
+ :model/Database {d2 :id} {:engine "h2", :details (:details (mt/db))}
+ :model/Table {t1 :id} {:db_id d1, :schema "PUBLIC"}
+ :model/Table {t2 :id} {:db_id d1, :schema "PUBLIC"}
+ :model/Table {_ :id} {:db_id d2, :schema "PUBLIC"}
+ :model/Table {t4 :id} {:db_id d2, :schema "PUBLIC"}
+ :model/Table {t5 :id} {:db_id d2, :schema "FOO"}]
+ (with-redefs [quick-task/executor (delay pool)]
+ (mt/with-dynamic-fn-redefs [sync/sync-table! (fn [table]
+ (swap! tables conj table)
+ (.countDown latch)
+ nil)]
+ (mt/user-http-request :crowberto :post 204 "data-studio/table/sync-schema" {:database_ids [d1],
+ :schema_ids [(format "%d:FOO" d2)]
+ :table_ids [t4]})
+ (testing "sync called?"
+ (is (true? (.await latch 4 TimeUnit/SECONDS)))
+ (is (= [t1 t2 t4 t5] (map :id @tables)))))))
+ (finally
+ (.shutdownNow pool))))))
(deftest ^:parallel non-admins-cant-trigger-bulk-rescan-values-test
(testing "Non-admins should not be allowed to trigger rescan values"
@@ -125,24 +136,31 @@
;; lot more to test here but will wait for firmer ground
(testing "Can we trigger a field values sync for a filtered set of tables"
(let [tables (atom [])
- latch (CountDownLatch. 4)]
- (mt/with-temp [:model/Database {d1 :id} {:engine "h2", :details (:details (mt/db))}
- :model/Database {d2 :id} {:engine "h2", :details (:details (mt/db))}
- :model/Table {t1 :id} {:db_id d1, :schema "PUBLIC"}
- :model/Table {t2 :id} {:db_id d1, :schema "PUBLIC"}
- :model/Table {_ :id} {:db_id d2, :schema "PUBLIC"}
- :model/Table {t4 :id} {:db_id d2, :schema "PUBLIC"}
- :model/Table {t5 :id} {:db_id d2, :schema "FOO"}]
- (mt/with-dynamic-fn-redefs [sync/update-field-values-for-table! (fn [table]
- (swap! tables conj table)
- (.countDown latch)
- nil)]
- (mt/user-http-request :crowberto :post 204 "data-studio/table/rescan-values" {:database_ids [d1],
- :schema_ids [(format "%d:FOO" d2)]
- :table_ids [t4]})
- (testing "rescanned?"
- (is (true? (.await latch 4 TimeUnit/SECONDS)))
- (is (= [t1 t2 t4 t5] (map :id @tables)))))))))
+ latch (CountDownLatch. 4)
+ ;; Isolated single-thread pool, for the same reasons as in
+ ;; `trigger-bulk-metadata-sync-for-table-test`.
+ pool (Executors/newSingleThreadExecutor)]
+ (try
+ (mt/with-temp [:model/Database {d1 :id} {:engine "h2", :details (:details (mt/db))}
+ :model/Database {d2 :id} {:engine "h2", :details (:details (mt/db))}
+ :model/Table {t1 :id} {:db_id d1, :schema "PUBLIC"}
+ :model/Table {t2 :id} {:db_id d1, :schema "PUBLIC"}
+ :model/Table {_ :id} {:db_id d2, :schema "PUBLIC"}
+ :model/Table {t4 :id} {:db_id d2, :schema "PUBLIC"}
+ :model/Table {t5 :id} {:db_id d2, :schema "FOO"}]
+ (with-redefs [quick-task/executor (delay pool)]
+ (mt/with-dynamic-fn-redefs [sync/update-field-values-for-table! (fn [table]
+ (swap! tables conj table)
+ (.countDown latch)
+ nil)]
+ (mt/user-http-request :crowberto :post 204 "data-studio/table/rescan-values" {:database_ids [d1],
+ :schema_ids [(format "%d:FOO" d2)]
+ :table_ids [t4]})
+ (testing "rescanned?"
+ (is (true? (.await latch 4 TimeUnit/SECONDS)))
+ (is (= [t1 t2 t4 t5] (map :id @tables)))))))
+ (finally
+ (.shutdownNow pool))))))
(deftest ^:parallel non-admins-cant-trigger-bulk-discard-values-test
(testing "Non-admins should not be allowed to trigger discard values"
diff --git a/test/metabase/warehouse_schema_rest/api/table_test.clj b/test/metabase/warehouse_schema_rest/api/table_test.clj
index d5758b18148..a1b513b8ed7 100644
--- a/test/metabase/warehouse_schema_rest/api/table_test.clj
+++ b/test/metabase/warehouse_schema_rest/api/table_test.clj
@@ -24,7 +24,9 @@
[metabase.util :as u]
[metabase.util.quick-task :as quick-task]
[metabase.warehouse-schema-rest.api.table :as api.table]
- [toucan2.core :as t2]))
+ [toucan2.core :as t2])
+ (:import
+ (java.util.concurrent Executors)))
(set! *warn-on-reflection* true)
@@ -1290,18 +1292,27 @@
(testing "POST /api/table/:id/sync_schema"
(testing "User with manage-table-metadata permission can sync table"
(let [sync-called? (promise)
- timeout (* 60 1000)]
- (mt/with-premium-features #{:audit-app}
- (mt/with-temp [:model/Database {db-id :id} {:engine "h2", :details (:details (mt/db))}
- :model/Table table {:db_id db-id :schema "PUBLIC"}]
- (mt/with-no-data-perms-for-all-users!
- ;; Grant only manage-table-metadata permission for this table
- (data-perms/set-table-permission! (perms-group/all-users) (:id table) :perms/manage-table-metadata :yes)
- (with-redefs [sync/sync-table! (deliver-when-tbl sync-called? table)]
- (mt/user-http-request :rasta :post 200 (format "table/%d/sync_schema" (u/the-id table)))
- (testing "sync called?"
- (is (true?
- (deref sync-called? timeout :sync-never-called))))))))))))
+ timeout (* 60 1000)
+ ;; Isolated pool: the shared one is process-wide and holds fire-and-forget tasks left behind by
+ ;; earlier tests, each with the default two-hour timeout. One of those still running ahead of
+ ;; this sync starves it past the deref below, which is what happens on driver CI, where those
+ ;; leftover tasks are real syncs over the network.
+ pool (Executors/newSingleThreadExecutor)]
+ (try
+ (mt/with-premium-features #{:audit-app}
+ (mt/with-temp [:model/Database {db-id :id} {:engine "h2", :details (:details (mt/db))}
+ :model/Table table {:db_id db-id :schema "PUBLIC"}]
+ (mt/with-no-data-perms-for-all-users!
+ ;; Grant only manage-table-metadata permission for this table
+ (data-perms/set-table-permission! (perms-group/all-users) (:id table) :perms/manage-table-metadata :yes)
+ (with-redefs [quick-task/executor (delay pool)]
+ (mt/with-dynamic-fn-redefs [sync/sync-table! (deliver-when-tbl sync-called? table)]
+ (mt/user-http-request :rasta :post 200 (format "table/%d/sync_schema" (u/the-id table)))
+ (testing "sync called?"
+ (is (true?
+ (deref sync-called? timeout :sync-never-called)))))))))
+ (finally
+ (.shutdownNow pool)))))))
(deftest sync-schema-mirror-database-test
(testing "POST /api/table/:id/sync_schema"
diff --git a/test/metabase/warehouses_rest/api_test.clj b/test/metabase/warehouses_rest/api_test.clj
index b9fafe4426b..1763e3b768f 100644
--- a/test/metabase/warehouses_rest/api_test.clj
+++ b/test/metabase/warehouses_rest/api_test.clj
@@ -51,7 +51,7 @@
[toucan2.core :as t2])
(:import
(java.sql Connection)
- (java.util.concurrent CountDownLatch)
+ (java.util.concurrent CountDownLatch Executors)
(org.quartz JobDetail TriggerKey)))
(set! *warn-on-reflection* true)
@@ -1735,9 +1735,15 @@
(deftest sync-schema-executes-when-executor-busy-test
(testing "POST /api/database/:id/sync_schema should execute sync even when quick-task executor is busy (GHY-3254)"
(let [sync-called? (promise)
- blocker-latch (CountDownLatch. 1)]
+ blocker-latch (CountDownLatch. 1)
+ ;; Run on a pool of our own: the shared one is process-wide and holds fire-and-forget tasks left
+ ;; behind by earlier tests, each with the default two-hour timeout. One of those still running
+ ;; ahead of the blocker delays everything below it past the deref timeout, which is what happens
+ ;; on driver CI, where those leftover tasks are real syncs over the network.
+ pool (Executors/newSingleThreadExecutor)]
(mt/with-temp [:model/Database {db-id :id} {:engine "h2" :details (:details (mt/db))}]
- (with-redefs [sync-metadata/sync-db-metadata-explicit! (deliver-when-db sync-called? db-id)
+ (with-redefs [quick-task/executor (delay pool)
+ sync-metadata/sync-db-metadata-explicit! (deliver-when-db sync-called? db-id)
analyze/analyze-db-explicit! (constantly nil)]
;; Submit a blocking task with a 1-second timeout so it gets cancelled quickly.
;; This simulates a stuck sync (e.g., hanging JDBC connection) that exceeds
@@ -1751,7 +1757,8 @@
(testing "sync executes after stuck task is evicted"
(is (true? (deref sync-called? 10000 :sync-never-called))))
(finally
- (.countDown blocker-latch))))))))
+ (.countDown blocker-latch)
+ (.shutdownNow pool))))))))
(deftest ^:parallel dismiss-spinner-test
(testing "Can we dismiss the spinner? (#20863)"
@@ -1778,22 +1785,28 @@
(deftest can-rescan-fieldvalues-for-a-db
(testing "Can we RESCAN all the FieldValues for a DB?"
(mt/with-premium-features #{:audit-app}
- (let [update-field-values-called? (promise)]
- (mt/with-temp [:model/Database db {:engine "h2", :details (:details (mt/db))}]
- (mt/with-dynamic-fn-redefs [sync.field-values/update-field-values! (fn [synced-db]
- (when (= (u/the-id synced-db) (u/the-id db))
- (deliver update-field-values-called? :sync-called)))]
- (snowplow-test/with-fake-snowplow-collector
- (mt/user-http-request :crowberto :post 200 (format "database/%d/rescan_values" (u/the-id db)))
- (is (= :sync-called
- (deref update-field-values-called? long-timeout :sync-never-called)))
- (is (= (:id db) (:model_id (mt/latest-audit-log-entry "database-manual-scan"))))
- (is (= (:id db) (-> (mt/latest-audit-log-entry "database-manual-scan")
- :details :id)))
- (testing "triggers snowplow event"
- (is (=?
- {"event" "database_manual_scan", "target_id" (u/the-id db)}
- (:data (last (snowplow-test/pop-event-data-and-user-id!)))))))))))))
+ (let [update-field-values-called? (promise)
+ ;; Isolated pool, for the same reasons as in `sync-schema-executes-when-executor-busy-test`.
+ pool (Executors/newSingleThreadExecutor)]
+ (try
+ (mt/with-temp [:model/Database db {:engine "h2", :details (:details (mt/db))}]
+ (with-redefs [quick-task/executor (delay pool)]
+ (mt/with-dynamic-fn-redefs [sync.field-values/update-field-values! (fn [synced-db]
+ (when (= (u/the-id synced-db) (u/the-id db))
+ (deliver update-field-values-called? :sync-called)))]
+ (snowplow-test/with-fake-snowplow-collector
+ (mt/user-http-request :crowberto :post 200 (format "database/%d/rescan_values" (u/the-id db)))
+ (is (= :sync-called
+ (deref update-field-values-called? long-timeout :sync-never-called)))
+ (is (= (:id db) (:model_id (mt/latest-audit-log-entry "database-manual-scan"))))
+ (is (= (:id db) (-> (mt/latest-audit-log-entry "database-manual-scan")
+ :details :id)))
+ (testing "triggers snowplow event"
+ (is (=?
+ {"event" "database_manual_scan", "target_id" (u/the-id db)}
+ (:data (last (snowplow-test/pop-event-data-and-user-id!))))))))))
+ (finally
+ (.shutdownNow pool)))))))
(deftest ^:parallel nonadmins-cant-trigger-rescan-test
(testing "Non-admins should not be allowed to trigger re-scan"