Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Fix usage_info IN clause exceeding 65K parameter limit (GHY-2413) (#71856)

Eric Normand74a08a7a2df3
diff --git a/src/metabase/warehouses_rest/api.clj b/src/metabase/warehouses_rest/api.clj
index 505f14422d2..d726db7ba2a 100644
--- a/src/metabase/warehouses_rest/api.clj
+++ b/src/metabase/warehouses_rest/api.clj
@@ -489,17 +489,11 @@
"List of models that are used to report usage on a database."
[:question :dataset :metric :segment]) ; TODO -- rename `:dataset` to `:model`?
-(def ^:private always-false-hsql-expr
- "A Honey SQL expression that is never true.
-
- 1 = 2"
- [:= [:inline 1] [:inline 2]])
-
(defmulti ^:private database-usage-query
"Query that will returns the number of `model` that use the database with id `database-id`.
The query must returns a scalar, and the method could return `nil` in case no query is available."
- {:arglists '([model database-id table-ids])}
- (fn [model _database-id _table-ids] (keyword model)))
+ {:arglists '([model database-id])}
+ (fn [model _database-id] (keyword model)))
(defn- card-query
[db-id model type-str]
@@ -510,24 +504,24 @@
[:= :type type-str]]})
(defmethod database-usage-query :question
- [_ db-id _table-ids]
+ [_ db-id]
(card-query db-id :question "question"))
(defmethod database-usage-query :dataset
- [_model db-id _table-ids]
+ [_ db-id]
(card-query db-id :dataset "model"))
(defmethod database-usage-query :metric
- [_ db-id _table-ids]
+ [_ db-id]
(card-query db-id :metric "metric"))
(defmethod database-usage-query :segment
- [_ _db-id table-ids]
+ [_ db-id]
{:select [[:%count.* :segment]]
:from [:segment]
- :where (if table-ids
- [:in :table_id table-ids]
- always-false-hsql-expr)})
+ :where [:in :table_id {:select [:id]
+ :from [:metabase_table]
+ :where [:= :db_id db-id]}]})
;; TODO (Cam 10/28/25) -- fix this endpoint route to use kebab-case for consistency with the rest of our REST API
;;
@@ -543,13 +537,12 @@
[:id ms/PositiveInt]]]
(api/check-superuser)
(check-database-exists id)
- (let [table-ids (t2/select-pks-set :model/Table :db_id id)]
- (first (mdb/query
- {:select [:*]
- :from (for [model database-usage-models
- :let [query (database-usage-query model id table-ids)]
- :when query]
- [query model])}))))
+ (first (mdb/query
+ {:select [:*]
+ :from (for [model database-usage-models
+ :let [query (database-usage-query model id)]
+ :when query]
+ [query model])})))
;;; ----------------------------------------- GET /api/database/:id/metadata -----------------------------------------
diff --git a/test/metabase/warehouses_rest/api_test.clj b/test/metabase/warehouses_rest/api_test.clj
index 7d4c5499854..c302b91a3f7 100644
--- a/test/metabase/warehouses_rest/api_test.clj
+++ b/test/metabase/warehouses_rest/api_test.clj
@@ -3,9 +3,11 @@
(:require
[clojure.string :as str]
[clojure.test :refer :all]
+ [clojure.walk :as walk]
[clojurewerkz.quartzite.scheduler :as qs]
[medley.core :as m]
[metabase.analytics.snowplow-test :as snowplow-test]
+ [metabase.app-db.core :as mdb]
[metabase.audit-app.core :as audit]
[metabase.config.core :as config]
[metabase.driver :as driver]
@@ -278,6 +280,39 @@
(mt/user-http-request :crowberto :get 404
(format "database/%d/usage_info" non-existing-db-id))))))))
+(defn- find-in-clauses
+ "Walk a HoneySQL map and return any [:in ...] clauses where the value is a collection."
+ [hsql]
+ (let [results (volatile! [])]
+ (walk/postwalk
+ (fn [form]
+ (when (and (vector? form)
+ (let [[op _ coll] form]
+ (and
+ (= :in op)
+ (= 3 (count form))
+ (coll? coll)
+ (not (map? coll)))))
+ (vswap! results conj form))
+ form)
+ hsql)
+ @results))
+
+(deftest get-database-usage-info-no-large-in-test
+ (testing "usage_info query should not use IN clauses with more than 100 items (GHY-2413)"
+ (mt/with-temp
+ [:model/Database {db-id :id} {}
+ :model/Table _ {:db_id db-id}]
+ (let [queries (volatile! [])
+ orig-query mdb/query]
+ (with-redefs [mdb/query (fn [hsql]
+ (vswap! queries conj hsql)
+ (orig-query hsql))]
+ (mt/user-http-request :crowberto :get 200 (format "database/%d/usage_info" db-id)))
+ (doseq [q @queries]
+ (is (empty? (find-in-clauses q))
+ "usage_info should not generate IN clauses with inline collections"))))))
+
(deftest ^:parallel get-database-usage-info-test-2
(mt/with-temp
[:model/Database {db-id :id} {}]