Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Fix sync_schema silently dropping when executor is busy (GHY-3254) (#71756)

Eric Normand23410f5c244e
diff --git a/src/metabase/util/quick_task.clj b/src/metabase/util/quick_task.clj
index cf9c1e16a11..f40b4318ff5 100644
--- a/src/metabase/util/quick_task.clj
+++ b/src/metabase/util/quick_task.clj
@@ -2,9 +2,13 @@
"Namespace with helpers for quick tasks. Intended for quick, one-off tasks like re-syncing a table,
fingerprinting a field, etc."
(:require
- [metabase.classloader.core :as classloader])
+ [clojure.string :as str]
+ [metabase.classloader.core :as classloader]
+ [metabase.config.core :as config]
+ [metabase.util :as u]
+ [metabase.util.log :as log])
(:import
- (java.util.concurrent Callable Executors ExecutorService Future ThreadFactory)))
+ (java.util.concurrent Callable Executors ExecutorService Future FutureTask ThreadFactory TimeUnit TimeoutException)))
(set! *warn-on-reflection* true)
@@ -20,10 +24,34 @@
(defonce ^:private executor
(delay (Executors/newFixedThreadPool 1 ^ThreadFactory thread-factory)))
+(defn task-timeout-ms
+ "Returns the task timeout in milliseconds. Reads from the `MB_QUICK_TASK_TIMEOUT_MINUTES` env var,
+ defaulting to 120 minutes (2 hours)."
+ []
+ (u/minutes->ms (or (config/config-int :mb-quick-task-timeout-minutes) 120)))
+
(defn submit-task!
- "Submit a task to the single thread executor. This will attempt to serialize repeated requests to sync tables. It
- obviously cannot work across multiple instances."
+ "Submit a task to the single thread executor. Each task is run on a separate thread with a timeout
+ controlled by the `MB_QUICK_TASK_TIMEOUT_MINUTES` env var (default 120 minutes). If a task exceeds
+ the timeout, it is cancelled and the executor moves on to the next queued task. This prevents a
+ single stuck task from blocking all subsequent tasks."
^Future [^Callable f]
{:pre [(some? f)]}
- (let [task (bound-fn* f)]
- (.submit ^ExecutorService @executor ^Callable task)))
+ (let [timeout-ms (long (task-timeout-ms))
+ task (bound-fn* f)
+ wrapped (fn []
+ (let [fut (FutureTask. ^Callable task)
+ t (.newThread ^ThreadFactory thread-factory ^Runnable fut)]
+ (.start t)
+ (try
+ (.get fut timeout-ms TimeUnit/MILLISECONDS)
+ (catch TimeoutException _
+ (log/warnf "quick-task exceeded timeout of %d ms, cancelling. Stuck thread stack trace:\n%s"
+ timeout-ms
+ (str/join "\n" (map #(str " " %) (.getStackTrace t))))
+ (.cancel fut true))
+ (catch InterruptedException _
+ (.interrupt (Thread/currentThread)))
+ (catch Exception e
+ (log/warn e "quick-task threw exception")))))]
+ (.submit ^ExecutorService @executor ^Callable wrapped)))
diff --git a/test/metabase/warehouses_rest/api_test.clj b/test/metabase/warehouses_rest/api_test.clj
index e4775eb4d25..7d4c5499854 100644
--- a/test/metabase/warehouses_rest/api_test.clj
+++ b/test/metabase/warehouses_rest/api_test.clj
@@ -47,6 +47,7 @@
[toucan2.core :as t2])
(:import
(java.sql Connection)
+ (java.util.concurrent CountDownLatch)
(org.quartz JobDetail TriggerKey)))
(set! *warn-on-reflection* true)
@@ -1514,6 +1515,27 @@
{"event" "database_manual_sync", "target_id" db-id}
(:data (last (snowplow-test/pop-event-data-and-user-id!)))))))))))))
+(deftest sync-schema-executes-when-executor-busy-test
+ (testing "POST /api/database/:id/sync_schema should execute sync even when quick-task executor is busy (GHY-3254)"
+ (let [sync-called? (promise)
+ blocker-latch (CountDownLatch. 1)]
+ (mt/with-temp [:model/Database {db-id :id} {:engine "h2" :details (:details (mt/db))}]
+ (with-redefs [sync-metadata/sync-db-metadata! (deliver-when-db sync-called? db-id)
+ analyze/analyze-db! (constantly nil)]
+ ;; Submit a blocking task with a 1-second timeout so it gets cancelled quickly.
+ ;; This simulates a stuck sync (e.g., hanging JDBC connection) that exceeds
+ ;; the quick-task timeout and gets evicted.
+ (with-redefs [quick-task/task-timeout-ms (constantly 1000)]
+ (quick-task/submit-task! (fn [] (.await blocker-latch))))
+ (try
+ (mt/user-http-request :crowberto :post 200 (format "database/%d/sync_schema" db-id))
+ ;; The sync task is queued behind the blocker. After the blocker times out
+ ;; and is cancelled, the sync task should execute.
+ (testing "sync executes after stuck task is evicted"
+ (is (true? (deref sync-called? 10000 :sync-never-called))))
+ (finally
+ (.countDown blocker-latch))))))))
+
(deftest ^:parallel dismiss-spinner-test
(testing "Can we dismiss the spinner? (#20863)"
(mt/with-temp [:model/Database db {:engine "h2", :details (:details (mt/db)) :initial_sync_status "incomplete"}