The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Fix sync_schema silently dropping when executor is busy (GHY-3254) (#71756)
Eric Normand23410f5c244e
diff --git a/src/metabase/util/quick_task.clj b/src/metabase/util/quick_task.cljindex cf9c1e16a11..f40b4318ff5 100644--- a/src/metabase/util/quick_task.clj+++ b/src/metabase/util/quick_task.clj@@ -2,9 +2,13 @@"Namespace with helpers for quick tasks. Intended for quick, one-off tasks like re-syncing a table,fingerprinting a field, etc."(:require- [metabase.classloader.core :as classloader])+ [clojure.string :as str]+ [metabase.classloader.core :as classloader]+ [metabase.config.core :as config]+ [metabase.util :as u]+ [metabase.util.log :as log])(:import- (java.util.concurrent Callable Executors ExecutorService Future ThreadFactory)))+ (java.util.concurrent Callable Executors ExecutorService Future FutureTask ThreadFactory TimeUnit TimeoutException)))(set! *warn-on-reflection* true)@@ -20,10 +24,34 @@(defonce ^:private executor(delay (Executors/newFixedThreadPool 1 ^ThreadFactory thread-factory)))+(defn task-timeout-ms+ "Returns the task timeout in milliseconds. Reads from the `MB_QUICK_TASK_TIMEOUT_MINUTES` env var,+ defaulting to 120 minutes (2 hours)."+ []+ (u/minutes->ms (or (config/config-int :mb-quick-task-timeout-minutes) 120)))+(defn submit-task!- "Submit a task to the single thread executor. This will attempt to serialize repeated requests to sync tables. It- obviously cannot work across multiple instances."+ "Submit a task to the single thread executor. Each task is run on a separate thread with a timeout+ controlled by the `MB_QUICK_TASK_TIMEOUT_MINUTES` env var (default 120 minutes). If a task exceeds+ the timeout, it is cancelled and the executor moves on to the next queued task. This prevents a+ single stuck task from blocking all subsequent tasks."^Future [^Callable f]{:pre [(some? f)]}- (let [task (bound-fn* f)]- (.submit ^ExecutorService @executor ^Callable task)))+ (let [timeout-ms (long (task-timeout-ms))+ task (bound-fn* f)+ wrapped (fn []+ (let [fut (FutureTask. ^Callable task)+ t (.newThread ^ThreadFactory thread-factory ^Runnable fut)]+ (.start t)+ (try+ (.get fut timeout-ms TimeUnit/MILLISECONDS)+ (catch TimeoutException _+ (log/warnf "quick-task exceeded timeout of %d ms, cancelling. Stuck thread stack trace:\n%s"+ timeout-ms+ (str/join "\n" (map #(str " " %) (.getStackTrace t))))+ (.cancel fut true))+ (catch InterruptedException _+ (.interrupt (Thread/currentThread)))+ (catch Exception e+ (log/warn e "quick-task threw exception")))))]+ (.submit ^ExecutorService @executor ^Callable wrapped)))diff --git a/test/metabase/warehouses_rest/api_test.clj b/test/metabase/warehouses_rest/api_test.cljindex e4775eb4d25..7d4c5499854 100644--- a/test/metabase/warehouses_rest/api_test.clj+++ b/test/metabase/warehouses_rest/api_test.clj@@ -47,6 +47,7 @@[toucan2.core :as t2])(:import(java.sql Connection)+ (java.util.concurrent CountDownLatch)(org.quartz JobDetail TriggerKey)))(set! *warn-on-reflection* true)@@ -1514,6 +1515,27 @@{"event" "database_manual_sync", "target_id" db-id}(:data (last (snowplow-test/pop-event-data-and-user-id!)))))))))))))+(deftest sync-schema-executes-when-executor-busy-test+ (testing "POST /api/database/:id/sync_schema should execute sync even when quick-task executor is busy (GHY-3254)"+ (let [sync-called? (promise)+ blocker-latch (CountDownLatch. 1)]+ (mt/with-temp [:model/Database {db-id :id} {:engine "h2" :details (:details (mt/db))}]+ (with-redefs [sync-metadata/sync-db-metadata! (deliver-when-db sync-called? db-id)+ analyze/analyze-db! (constantly nil)]+ ;; Submit a blocking task with a 1-second timeout so it gets cancelled quickly.+ ;; This simulates a stuck sync (e.g., hanging JDBC connection) that exceeds+ ;; the quick-task timeout and gets evicted.+ (with-redefs [quick-task/task-timeout-ms (constantly 1000)]+ (quick-task/submit-task! (fn [] (.await blocker-latch))))+ (try+ (mt/user-http-request :crowberto :post 200 (format "database/%d/sync_schema" db-id))+ ;; The sync task is queued behind the blocker. After the blocker times out+ ;; and is cancelled, the sync task should execute.+ (testing "sync executes after stuck task is evicted"+ (is (true? (deref sync-called? 10000 :sync-never-called))))+ (finally+ (.countDown blocker-latch))))))))+(deftest ^:parallel dismiss-spinner-test(testing "Can we dismiss the spinner? (#20863)"(mt/with-temp [:model/Database db {:engine "h2", :details (:details (mt/db)) :initial_sync_status "incomplete"}