The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
GHY-2937: On-demand field values can take forever to load (#81753)
Eric Normand0dfca163f0f5
diff --git a/enterprise/backend/test/metabase_enterprise/advanced_permissions/models/field_values_test.clj b/enterprise/backend/test/metabase_enterprise/advanced_permissions/models/field_values_test.cljindex bd16e253951..7f516d4a57f 100644--- a/enterprise/backend/test/metabase_enterprise/advanced_permissions/models/field_values_test.clj+++ b/enterprise/backend/test/metabase_enterprise/advanced_permissions/models/field_values_test.clj@@ -8,6 +8,7 @@[metabase.parameters.field-values :as params.field-values][metabase.test :as mt][metabase.util :as u]+ [metabase.warehouse-schema.models.field-values :as field-values][toucan2.core :as t2]))(deftest get-or-create-field-values!-test@@ -30,6 +31,15 @@(testing "calling a second time shouldn't create new FieldValues"(params.field-values/get-or-create-field-values! field)(is (= 1 (t2/count :model/FieldValues :field_id (u/the-id field) :type :advanced))))+ (testing "and a warm hit is served without a detached fetch — it is one SELECT, so+ handing it to a background thread costs more than it saves (GHY-2937)"+ (let [detached (atom 0)+ real field-values/detached-fetch!]+ (with-redefs [field-values/detached-fetch! (fn [& args]+ (swap! detached inc)+ (apply real args))]+ (params.field-values/get-or-create-field-values! field)+ (is (zero? @detached)))))(testing "changing the impersonation role creates new FieldValues"(impersonation.util-test/with-impersonations! {:impersonations [{:db-id (mt/id) :attribute "impersonation_attr"}]:attributes {"impersonation_attr" "impersonation_role_2"}}diff --git a/src/metabase/parameters/db.clj b/src/metabase/parameters/db.cljindex 06c2d55a2ef..9f84f968a92 100644--- a/src/metabase/parameters/db.clj+++ b/src/metabase/parameters/db.clj@@ -115,6 +115,11 @@[id](t2/delete! :model/FieldValues :id id))+(defn advanced-field-values+ "The advanced FieldValues of the Field with `field-id` and `hash-key`, or nil."+ [field-id hash-key]+ (t2/select-one :model/FieldValues :field_id field-id, :type :advanced, :hash_key hash-key))+(defn find-or-insert-advanced-field-values!"The advanced FieldValues of the Field with `field-id` and `hash-key`, inserting one built by calling`insert-fn` if none exists yet."diff --git a/src/metabase/parameters/field_values.clj b/src/metabase/parameters/field_values.cljindex cd11c0f5efb..e1036c15368 100644--- a/src/metabase/parameters/field_values.clj+++ b/src/metabase/parameters/field_values.clj@@ -145,8 +145,15 @@advanced-field-value? (not= hash-input {:field-id (u/the-id field)})](if advanced-field-value?(let [hash-key (str (hash hash-input))- fv (parameters.db/find-or-insert-advanced-field-values!- (:id field) hash-key #(prepare-advanced-field-values field hash-key constraints))]+ ;; look first on this thread: a hit is one SELECT, and handing that to a background+ ;; thread costs more than it saves. Only a miss is worth detaching, because only a+ ;; miss scans the warehouse.+ fv (or (parameters.db/advanced-field-values (:id field) hash-key)+ (field-values/detached-fetch!+ [:advanced (:id field) hash-key]+ (fn []+ (parameters.db/find-or-insert-advanced-field-values!+ (:id field) hash-key #(prepare-advanced-field-values field hash-key constraints)))))];; If it's expired, delete then try to re-create it(if (some-> fv field-values/advanced-field-values-expired?)(dodiff --git a/src/metabase/warehouse_schema/models/field_values.clj b/src/metabase/warehouse_schema/models/field_values.cljindex ae2375a7539..06094e891a4 100644--- a/src/metabase/warehouse_schema/models/field_values.clj+++ b/src/metabase/warehouse_schema/models/field_values.clj@@ -506,21 +506,155 @@"Create or update the full FieldValues object for `field`. If the FieldValues object alreadyexists, then update values for it; otherwise create a new FieldValues object with the newlyfetched values. Returns whether the field values were created / updated / deleted as a result- of this call.+ of this call, or `::fv-fetch-failed` if the warehouse scan failed and nothing was changed.Note that if the full FieldValues are create / updated / deleted, it'll delete all theAdvanced FieldValues of the same `field`."[field & {:keys [field-values]}](if (field-should-have-field-values? field)- (let [existing-fv (or field-values (get-latest-full-field-values (u/the-id field)))- {rows :values} (distinct-values field)- ;; rows are 1-tuples — unwrap for storage.- raw-values (seq (map first rows))]- (persist-field-values! field existing-fv raw-values))+ (let [existing-fv (or field-values (get-latest-full-field-values (u/the-id field)))]+ ;; `distinct-values` returns nil when the warehouse scan failed, which is not the same as a+ ;; field that genuinely has no values: persisting it would read as "empty" and delete the+ ;; values we already have. Leave whatever is cached in place instead.+ (if-let [{rows :values} (distinct-values field)]+ ;; rows are 1-tuples — unwrap for storage.+ (persist-field-values! field existing-fv (seq (map first rows)))+ ::fv-fetch-failed))(do(clear-field-values-for-field! field)::fv-deleted)))+(def ^:private in-flight-fetches+ "`cache-key` -> `{:promise :future-ref :timer}` for the fetch currently running for that key.+ See [[detached-fetch!]]."+ (atom {}))++(def ^:dynamic *fetch-max-age-ms*+ "How long a detached fetch may run before it is canceled, and how long a caller will wait on one.++ Only a fetch wedged outside the query processor can reach this: the warehouse query itself is+ already capped by `*query-timeout-ms*` (`MB_DB_QUERY_TIMEOUT_MINUTES`, 20 minutes in prod), so+ this is a backstop for work blocked somewhere the QP cannot cancel — acquiring a connection, or a+ driver that ignores cancellation.++ It bounds the wait as well as the work because [[sweep-stalled-fetches!]] only runs when a request+ arrives, and the case this guards against is precisely the one where none can: callers parked on a+ wedged fetch are request threads, so enough of them stop the server from serving anything at all.++ Deliberately a backstop rather than a request-latency bound. A value nearer a sensible HTTP+ timeout would hand the thread back far sooner, but returning before the values are ready changes+ what this endpoint promises its callers, and that is a decision of its own."+ (* 60 60 1000))++(def ^:dynamic *max-in-flight-fetches*+ "Ceiling on the registry, independent of whatever else happens to bound it.++ Callers park on their fetch, so the number in flight is today limited by the request thread pool+ — but that is an accident of how these endpoints are served, not a guarantee this namespace can+ make. Past this many, [[detached-fetch!]] refuses the fetch with a 503 rather than letting the+ registry grow: reaching this means something is wrong, and failing loudly beats absorbing it.++ Sized well clear of the request thread pool (50 by default), so an instance has to have raised+ `MB_JETTY_MAXTHREADS` well past its default before this can be reached at all."+ 200)++(defn- complete-fetch!+ "Drop `entry` from the registry and hand `result` to everyone waiting on it.++ Dropping the entry before delivering means a caller that wakes up and immediately asks again+ starts a fresh fetch rather than re-attaching to this finished one. The entry is removed only if+ it is still the registered one, so a fetch that was already swept can't evict its replacement."+ [cache-key entry result]+ (swap! in-flight-fetches (fn [m]+ (cond-> m+ (identical? entry (get m cache-key)) (dissoc cache-key))))+ (deliver (:promise entry) result))++(defn- sweep-stalled-fetches!+ "Cancel every fetch that has outlived [[*fetch-max-age-ms*]] and fail whoever is waiting on it.++ A fetch clears its own registry entry, so this fires only when that failed or when the work is+ genuinely wedged. It completes the entries rather than merely forgetting them: callers park on the+ promise, so an entry dropped without delivery leaves them blocked forever, each holding a request+ thread. Deliberately logs no cache keys — printing one is a way this could throw, and a backstop+ that throws is not a backstop."+ []+ (let [swept (atom 0)]+ (doseq [[cache-key {:keys [timer future-ref] :as entry}] @in-flight-fetches+ :when (> (u/since-ms timer) *fetch-max-age-ms*)]+ (some-> @future-ref future-cancel)+ (complete-fetch! cache-key entry+ {:error (ex-info (tru "Timed out fetching field values.") {:cache-key cache-key})})+ (swap! swept inc))+ (when (pos? @swept)+ (log/warnf "Canceled %d FieldValues fetch(es) still running after %d ms" @swept *fetch-max-age-ms*))))++(defn- claim-fetch!+ "Register `entry` for `cache-key` and return whichever entry is registered once we are done, or+ nil when the registry is already full.++ The `contains?` check comes first deliberately: joining a fetch already in flight adds no entry,+ so a full registry must not turn those callers away. Only a new key can be refused. Checking+ capacity inside the swap keeps the ceiling exact under concurrent claims."+ [cache-key entry]+ (-> (swap! in-flight-fetches (fn [m]+ (if (or (contains? m cache-key)+ (>= (count m) *max-in-flight-fetches*))+ m+ (assoc m cache-key entry))))+ (get cache-key)))++(defn detached-fetch!+ "Run `thunk` on a background thread and return its result, rethrowing anything it throws.++ Calls sharing a `cache-key` while a fetch is in flight all wait on that one run instead of+ starting their own. Because the work is detached from the calling thread, it runs to completion+ — and persists whatever it fetched — even when the caller stops waiting, e.g. when the HTTP+ request that asked for the values is canceled."+ [cache-key thunk]+ ;; sweep here rather than on a timer: the registry only grows when something is inserted, so this+ ;; runs exactly when it needs to. Guarded because a failure to sweep must not fail this caller.+ (try+ (sweep-stalled-fetches!)+ (catch Throwable e+ (log/warn e "Error sweeping stalled FieldValues fetches")))+ (let [entry {:promise (promise), :future-ref (atom nil), :timer (u/start-timer)}+ this (or (claim-fetch! cache-key entry)+ (throw (ex-info (tru "Too many field values fetches are already running. Please try again shortly.")+ {:status-code 503+ :max-in-flight *max-in-flight-fetches*})))]+ (when (identical? this entry)+ ;; Every path from here must reach `complete-fetch!`. An entry left in the registry with its+ ;; promise undelivered is worse than a leak: every later caller for that key parks on it+ ;; forever, holding a request thread each.+ (try+ (reset! (:future-ref entry)+ (future+ (try+ (complete-fetch! cache-key entry {:value (thunk)})+ (catch Throwable e+ (complete-fetch! cache-key entry {:error e})+ ;; log only once everyone waiting has been served: an appender that throws here+ ;; would otherwise strand them. Log at all because when every caller has walked+ ;; away there is nobody left to deref the promise, so this is the only record.+ (log/warnf e "Error fetching FieldValues for %s" (pr-str cache-key))))))+ ;; submitting can fail on its own — `future`'s pool rejects new work once the JVM starts+ ;; shutting down, and the registry entry is already in place by then+ (catch Throwable e+ (complete-fetch! cache-key entry {:error e}))))+ (let [{:keys [value error] :as result} (deref (:promise this) *fetch-max-age-ms* ::timed-out)]+ (cond+ (= result ::timed-out)+ (throw (ex-info (tru "Timed out waiting for field values.")+ {:status-code 503+ :timeout-ms *fetch-max-age-ms*}))++ error+ (throw error)++ :else+ value))))+(defn get-or-create-full-field-values!"Create FieldValues for a `Field` if they *should* exist but don't already exist. Returns the existing or newlycreated FieldValues for `Field`. Updates :last_used_at so sync will know this is active."@@ -529,17 +663,20 @@(when (field-should-have-field-values? field)(let [existing (or (not-empty field-values) (get-latest-full-field-values field-id))](if (or (not existing) (inactive? existing))- (case (create-or-update-full-field-values! field)- ::fv-deleted- nil-- ::fv-created- (get-latest-full-field-values field-id)-- (do- (when existing- (warehouse-schema.db/touch-field-values! (:id existing)))- (get-latest-full-field-values field-id)))+ (detached-fetch!+ [:full field-id]+ (fn []+ (case (create-or-update-full-field-values! field)+ ::fv-deleted+ nil++ ::fv-created+ (get-latest-full-field-values field-id)++ (do+ (when existing+ (warehouse-schema.db/touch-field-values! (:id existing)))+ (get-latest-full-field-values field-id)))))(do(warehouse-schema.db/touch-field-values! (:id existing))existing)))))diff --git a/src/metabase/warehouse_schema_rest/api/field.clj b/src/metabase/warehouse_schema_rest/api/field.cljindex 82da092fd7b..4ade23ac14a 100644--- a/src/metabase/warehouse_schema_rest/api/field.clj+++ b/src/metabase/warehouse_schema_rest/api/field.clj@@ -338,12 +338,16 @@[{:keys [id]} :- [:map[:id ms/PositiveInt]]](analytics/track-event! :snowplow/simple_event {:event "field_manual_scan" :target_id id})- (let [field (api/write-check (warehouse-schema-rest.db/field id))]- ;; Grant full permissions so that permission checks pass during sync. If a user has DB detail perms- ;; but no data perms, they should stll be able to trigger a sync of field values. This is fine because we don't- ;; return any actual field values from this API. (#21764)- (request/as-admin- (field-values/create-or-update-full-field-values! field)))+ (let [field (api/write-check (warehouse-schema-rest.db/field id))+ ;; Grant full permissions so that permission checks pass during sync. If a user has DB detail perms+ ;; but no data perms, they should stll be able to trigger a sync of field values. This is fine because we don't+ ;; return any actual field values from this API. (#21764)+ result (request/as-admin+ (field-values/create-or-update-full-field-values! field))]+ ;; a scan that fails leaves the existing FieldValues untouched, so reporting success would give the+ ;; caller no way at all to tell it happened. The underlying error is already in the server logs.+ (api/check (not= ::field-values/fv-fetch-failed result)+ [500 (i18n/tru "Failed to scan field values. Check the server logs for the underlying error.")])){:status :success});; TODO (Cam 10/28/25) -- fix this endpoint route to use kebab-case for consistency with the rest of our REST APIdiff --git a/test/metabase/warehouse_schema/models/field_values_test.clj b/test/metabase/warehouse_schema/models/field_values_test.cljindex cfa88c11d89..46c31f64d79 100644--- a/test/metabase/warehouse_schema/models/field_values_test.clj+++ b/test/metabase/warehouse_schema/models/field_values_test.clj@@ -13,6 +13,7 @@[metabase.sync.core :as sync][metabase.test :as mt][metabase.test.fixtures :as fixtures]+ [metabase.test.util :as tu][metabase.util :as u][metabase.util.json :as json][metabase.warehouse-schema.field-values.distinct-batch :as distinct-batch]@@ -22,6 +23,8 @@(:import(clojure.lang ExceptionInfo)))+(set! *warn-on-reflection* true)+(use-fixtures :once (fixtures/initialize :db))(def ^:private base-types-without-field-values@@ -266,6 +269,162 @@(is (not= (t/offset-date-time 2001 12)(:last_used_at (t2/select-one :model/FieldValues :field_id (mt/id :categories :name) :type :full))))))))+(deftest detached-fetch!-shares-in-flight-work-test+ (testing "callers sharing a cache key wait on the one in-flight run instead of each starting their own,+ so retrying a slow field values request does not pile up warehouse scans (GHY-2937)"+ (let [runs (atom 0)+ thunk (fn []+ (swap! runs inc)+ ;; hold the fetch open long enough that every caller reaches it while it is in flight+ (Thread/sleep 1000)+ ::values)+ callers (doall (repeatedly 5 #(future (field-values/detached-fetch! ::shared-key thunk))))]+ (is (= (repeat 5 ::values)+ (map #(deref % 30000 ::timed-out) callers)))+ (is (= 1 @runs)))))++(deftest detached-fetch!-outlives-canceled-caller-test+ (testing "a fetch runs to completion even after the caller stops waiting, e.g. when the HTTP request+ that asked for the values is canceled (GHY-2937)"+ (let [started (promise)+ release (promise)+ finished (promise)+ caller (future (field-values/detached-fetch!+ ::canceled-key+ (fn []+ (deliver started true)+ @release+ (deliver finished true))))]+ @started+ (is (true? (future-cancel caller)))+ (deliver release true)+ (is (true? (deref finished 10000 ::timed-out))))))++(deftest detached-fetch!-always-completes-test+ (testing "nothing may leave a registry entry undelivered — later callers for that key would park+ on it forever, holding a request thread each (GHY-2937)"+ ;; `detached-fetch!` pr-strs the cache key to log a failed fetch, so a key that refuses to print+ ;; makes the logging itself throw. That is the one path that used to escape between dropping the+ ;; registry entry and delivering the promise.+ (let [unprintable (reify Object (toString [_] (throw (ex-info "unprintable" {}))))+ registry @#'field-values/in-flight-fetches+ caller (future (try+ (field-values/detached-fetch! unprintable #(throw (ex-info "boom" {})))+ (catch Throwable _ ::threw)))]+ (is (= ::threw (deref caller 10000 ::timed-out)))+ (is (not (contains? @registry unprintable)))+ (testing "and the key is usable again afterwards"+ (is (= ::ok (field-values/detached-fetch! unprintable (constantly ::ok))))))))++(deftest detached-fetch!-sweeps-stalled-fetches-test+ (testing "a fetch that outlives the max age is canceled and its waiters are failed, so no registry+ entry can outlive its work (GHY-2937)"+ (let [registry @#'field-values/in-flight-fetches+ started (promise)+ release (promise)+ caller (future (try+ (field-values/detached-fetch! ::stalled (fn []+ (deliver started true)+ @release))+ (catch Throwable _ ::threw)))]+ @started+ (let [stalled-future @(:future-ref (get @registry ::stalled))]+ ;; backdate the entry's timer so the next call through detached-fetch! sees it as stalled.+ ;; Backdating this one entry rather than shortening the max age keeps the sweep from+ ;; touching fetches other tests may have in flight.+ (swap! registry update ::stalled update :timer - (* 24 60 60 1000 1000000))+ (testing "the sweep runs on the next fetch, which is unaffected by it"+ (is (= ::ok (field-values/detached-fetch! ::sweep-trigger (constantly ::ok)))))+ (is (= ::threw (deref caller 10000 ::timed-out)))+ (is (future-cancelled? stalled-future))+ (is (not (contains? @registry ::stalled)))))))++(deftest detached-fetch!-caps-registry-test+ (testing "past the registry cap the fetch is refused rather than growing the registry, so nothing+ outside this namespace has to bound it (GHY-2937)"+ (let [release (promise)+ started (promise)+ ran (atom false)]+ (try+ ;; a real in-flight fetch, so the joining case below goes through the public API rather than+ ;; a hand-built registry entry+ (let [holder (future (field-values/detached-fetch! ::held (fn []+ (deliver started true)+ @release)))]+ @started+ (binding [field-values/*max-in-flight-fetches* 0]+ (testing "a new key is refused with a 503, and its work never runs"+ (is (= 503 (try+ (field-values/detached-fetch! ::over-cap (fn [] (reset! ran true)))+ nil+ (catch clojure.lang.ExceptionInfo e+ (:status-code (ex-data e))))))+ (is (false? @ran)))+ (testing "but a caller joining a fetch already in flight is still admitted — it adds no+ registry entry, so the cap has no reason to turn it away"+ (let [joining (promise)+ joiner (future+ (deliver joining true)+ (field-values/detached-fetch! ::held (constantly ::should-not-run)))]+ @joining+ ;; the joiner has to reach the registry while ::held is still in flight; releasing the+ ;; holder first would let it complete, and the joiner would then be a new key the cap+ ;; refuses. It cannot return while `release` is undelivered, so a timeout here means+ ;; it parked on the held fetch.+ (is (= ::parked (deref joiner 1000 ::parked)))+ (deliver release ::from-held)+ (is (= ::from-held (deref joiner 10000 ::timed-out))))))+ (is (= ::from-held (deref holder 10000 ::timed-out))))+ (finally+ (deliver release ::done))))))++(deftest detached-fetch!-rethrows-test+ (testing "an exception thrown by the fetch reaches the caller"+ (is (thrown-with-msg? Exception #"oops"+ (field-values/detached-fetch! ::throwing-key #(throw (ex-info "oops" {})))))))++(deftest get-or-create-full-field-values!-outlives-canceled-caller-test+ (testing "FieldValues fetched on behalf of a canceled request still get saved to the app DB (GHY-2937)"+ (mt/dataset test-data+ (let [field-id (mt/id :categories :name)+ started (promise)+ release (promise)+ real-distinct-values (mt/original-fn #'field-values/distinct-values)]+ (t2/delete! :model/FieldValues :field_id field-id :type :full)+ (mt/with-dynamic-fn-redefs [field-values/distinct-values (fn [field]+ (deliver started true)+ @release+ (real-distinct-values field))]+ (let [caller (future (field-values/get-or-create-full-field-values!+ (t2/select-one :model/Field :id field-id)))]+ @started+ (is (true? (future-cancel caller)))+ (deliver release true)+ (is (seq (:values (tu/poll-until 10000+ (t2/select-one :model/FieldValues :field_id field-id :type :full)))))))))))++(deftest create-or-update-full-field-values!-fetch-failure-test+ (mt/dataset test-data+ (let [field-id (mt/id :categories :name)+ field (t2/select-one :model/Field :id field-id)+ cached #(t2/select-one :model/FieldValues :field_id field-id :type :full)]+ (field-values/get-or-create-full-field-values! field)+ (let [before (cached)]+ (is (seq (:values before)))+ (testing "a failed warehouse scan must not be read as \"this field has no values\" and wipe+ the FieldValues we already have cached (GHY-2937)"+ (mt/with-dynamic-fn-redefs [field-values/distinct-values (constantly nil)]+ (is (= ::field-values/fv-fetch-failed+ (field-values/create-or-update-full-field-values! field))))+ (is (= (:values before) (:values (cached)))))+ (testing "a scan that genuinely comes back empty still clears the FieldValues"+ (mt/with-dynamic-fn-redefs [field-values/distinct-values (constantly {:values []})]+ (is (= ::field-values/fv-deleted+ (field-values/create-or-update-full-field-values! field))))+ (is (nil? (cached))))+ ;; leave the shared test-data dataset as we found it+ (field-values/get-or-create-full-field-values! field)))))+(deftest normalize-human-readable-values-test(testing "If FieldValues were saved as a map, normalize them to a sequence on the way out"(mt/with-temp [:model/FieldValues fv {:field_id (mt/id :venues :id)diff --git a/test/metabase/warehouse_schema_rest/api/field_test.clj b/test/metabase/warehouse_schema_rest/api/field_test.cljindex 037bc69f79a..da0b237235e 100644--- a/test/metabase/warehouse_schema_rest/api/field_test.clj+++ b/test/metabase/warehouse_schema_rest/api/field_test.clj@@ -12,6 +12,7 @@[metabase.test.fixtures :as fixtures][metabase.util :as u][metabase.util.quick-task :as quick-task]+ [metabase.warehouse-schema.models.field-values :as field-values][toucan2.core :as t2]))(set! *warn-on-reflection* true)@@ -836,6 +837,19 @@[3 "Synergistic Granite Chair"]]}(mt/user-http-request :crowberto :get 200 (format "field/%d/values" (mt/id :orders :product_id)))))))))))+(deftest rescan-values-reports-scan-failure-test+ (testing "POST /api/field/:id/rescan_values"+ (testing "reports a failed scan rather than answering success — a failed scan leaves the cached+ FieldValues untouched, so there is otherwise no trace of it (GHY-2937)"+ (mt/with-temp-vals-in-db :model/Field (mt/id :venues :price) {:has_field_values "list"}+ ;; `with-redefs`, not `mt/with-dynamic-fn-redefs`: `mt/user-http-request` issues a real HTTP request to the+ ;; test Jetty server, which handles it on a worker thread that does not inherit this thread's dynamic+ ;; bindings. Only a root rebind is visible there.+ (with-redefs [field-values/distinct-values (constantly nil)]+ (is (=? {:message #"Failed to scan field values.*"}+ (mt/user-http-request :crowberto :post 500+ (format "field/%d/rescan_values" (mt/id :venues :price))))))))))+(deftest json-unfolding-initially-true-test(mt/test-drivers (mt/normal-drivers-with-feature :nested-field-columns)(when-not (mysql/mariadb? (mt/db))