Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Remove enhanced-database-tables and provide columnless tables in metabot context :used_tables (#76944)

appleby16eefb159d69
diff --git a/src/metabase/metabot/context.clj b/src/metabase/metabot/context.clj
index 7dc10f6776e..bb6bcd975dc 100644
--- a/src/metabase/metabot/context.clj
+++ b/src/metabase/metabot/context.clj
@@ -123,17 +123,30 @@
(when (lib/native-only-query? normalized-query)
normalized-query)))))
+(defn- table-stub
+ "Reference to a table used by a viewing-context item.
+
+ Excludes columns. [[metabase.metabot.agent.user-context/format-entity]] only reads `:type` and `:id` and calculates
+ column details via [[metabase.metabot.tools.entity-details/get-table-details]], so fetching columns here is wasted
+ work. On large, highly-connected schemas it can contribute to heap exhaustion (metabase#76493).
+ `:name`/`:database_schema`/`:description` are kept for the `format-simple-entity` fallback"
+ [{:keys [id name schema description]}]
+ {:id id
+ :type :table
+ :name name
+ :database_schema schema
+ :description description})
+
(defn- database-tables-for-context
- "Get database tables formatted for metabot context. Only includes tables used in the query, formatted for API output.
- Removes duplicate tables by id while preserving first occurrence order."
+ "Get database tables formatted for metabot context. Only includes tables used in the query.
+ Removes duplicate tables by id while preserving first occurrence order."
[{:keys [query]}]
(try
(if query
- (let [used-tables (table-utils/used-tables query)
- tables (table-utils/enhanced-database-tables (:database query)
- {:priority-tables used-tables
- :all-tables-limit (count used-tables)})]
- (m/distinct-by :id tables))
+ (into []
+ (comp (m/distinct-by :id)
+ (map table-stub))
+ (table-utils/used-tables query))
[])
(catch Exception e
(log/error e "Error getting database tables for context")
@@ -154,10 +167,7 @@
[{:keys [database-id table-ids]}]
(try
(when (and database-id (seq table-ids))
- (when-let [tables (not-empty (table-utils/used-tables-from-ids database-id table-ids))]
- (table-utils/enhanced-database-tables database-id
- {:priority-tables tables
- :all-tables-limit (count tables)})))
+ (not-empty (mapv table-stub (table-utils/used-tables-from-ids database-id table-ids))))
(catch Exception e
(log/error e "Error getting Python transform tables for context")
[])))
@@ -179,7 +189,7 @@
[database-id table-ids]))))
(defn- mbql-source-tables-for-context
- "Get source tables for an MBQL query, formatted for metabot context (with :type, :display_name, :fields, etc.).
+ "Get source tables for an MBQL query, formatted for metabot context.
Uses a direct table lookup without native-query permission checks. The user is already
viewing these tables in the notebook editor, so they have at least query-builder access.
@@ -187,16 +197,15 @@
which is too restrictive for MBQL viewing context enrichment."
[[database-id table-ids]]
(try
- (let [raw-tables (t2/select [:model/Table :id :name :schema]
+ (let [raw-tables (t2/select [:model/Table :id :name :schema :description]
:db_id database-id
:id [:in table-ids]
:active true
- :visibility_type nil)
- tables (when (seq raw-tables)
- (table-utils/enhanced-database-tables database-id
- {:priority-tables raw-tables
- :all-tables-limit (count raw-tables)}))]
- (m/distinct-by :id tables))
+ :visibility_type nil)]
+ (into []
+ (comp (m/distinct-by :id)
+ (map table-stub))
+ raw-tables))
(catch Exception e
(log/error e "Error getting MBQL source tables for context")
nil)))
diff --git a/src/metabase/metabot/table_utils.clj b/src/metabase/metabot/table_utils.clj
index db780537712..49442d539fb 100644
--- a/src/metabase/metabot/table_utils.clj
+++ b/src/metabase/metabot/table_utils.clj
@@ -3,14 +3,9 @@
(:require
[clojure.set :as set]
[metabase.api.common :as api]
- [metabase.lib-be.core :as lib-be]
- [metabase.lib.core :as lib]
- [metabase.lib.metadata :as lib.metadata]
[metabase.metabot.query-analyzer :as query-analyzer]
- [metabase.metabot.tools.util :as metabot.tools.u]
[metabase.models.interface :as mi]
[metabase.util :as u]
- [metabase.util.humanization :as u.humanization]
[toucan2.core :as t2]
[toucan2.realize :as t2.realize])
(:import
@@ -73,58 +68,6 @@
fields)}))
all-tables))))
-(defn enhanced-database-tables
- "Get database tables formatted with the new metabot tools schema format.
-
- Returns tables with :type, :display_name, :database_id, :database_schema, :fields (with field-id), :metrics.
- This format is used by metabot context and other modern tools."
- ([database-id]
- (enhanced-database-tables database-id nil))
- ([database-id {:keys [all-tables-limit priority-tables exclude-table-ids]
- :or {all-tables-limit max-database-tables
- priority-tables []
- exclude-table-ids #{}}}]
- (let [priority-table-ids (set (map :id priority-tables))
- {table-where-clause :clause table-cte :with} (mi/visible-filter-clause :model/Table
- :id
- {:user-id api/*current-user-id*
- :is-superuser? api/*is-superuser?*}
- {:perms/view-data :unrestricted
- :perms/create-queries :query-builder-and-native})
- ;; Fetch most viewed tables, excluding priority tables and excluded tables
- fill-tables (t2/select [:model/Table :id :db_id :name :schema :description]
- :db_id database-id
- :active true
- :visibility_type nil
- (cond-> {:where table-where-clause
- :order-by [[:view_count :desc]]
- :limit all-tables-limit}
- table-cte (assoc :with table-cte)))
- fill-tables (remove #(or (priority-table-ids (:id %))
- (exclude-table-ids (:id %))) fill-tables)
- all-tables (concat priority-tables fill-tables)
- all-tables (take all-tables-limit all-tables)]
- (lib-be/with-metadata-provider-cache
- (let [mp (lib-be/application-database-metadata-provider database-id)
- table-ids (map :id all-tables)
- _ (lib.metadata/bulk-metadata mp :metadata/table table-ids)
- engine (:engine (lib.metadata/database mp))]
- (mapv (fn [{:keys [id name schema description]}]
- (let [table-query (lib/query mp (lib.metadata/table mp id))
- cols (->> (lib/visible-columns table-query)
- (map #(metabot.tools.u/add-table-reference table-query %)))]
- {:id id
- :type :table
- :name name
- :display_name (u.humanization/name->human-readable-name :simple name)
- :database_id database-id
- :database_engine engine
- :database_schema schema
- :description description
- :fields (mapv #(metabot.tools.u/->result-column table-query %) cols)
- :metrics []}))
- all-tables))))))
-
(defn get-tables
"Get information about the tables in a given database.
@@ -198,7 +141,7 @@
(keep (fn [table]
(when (some #(matching-tables? table % {:match-schema? false}) unrecognized-tables)
(t2.realize/realize table))))
- (t2/reducible-select [:model/Table :id :name :schema]
+ (t2/reducible-select [:model/Table :id :name :schema :description]
:db_id database-id
:active true
:visibility_type nil
@@ -215,7 +158,7 @@
[database-id table-ids]
(if-not (seq table-ids)
[]
- (t2/select [:model/Table :id :name :schema]
+ (t2/select [:model/Table :id :name :schema :description]
:db_id database-id
:id [:in table-ids]
:active true
diff --git a/test/metabase/metabot/context_test.clj b/test/metabase/metabot/context_test.clj
index 833a496ad1a..516cbe585d4 100644
--- a/test/metabase/metabot/context_test.clj
+++ b/test/metabase/metabot/context_test.clj
@@ -16,82 +16,40 @@
(def ^:private users-native-query (lib/native-query meta/metadata-provider "SELECT * FROM users"))
(def ^:private users-mbql-query (lib/query meta/metadata-provider (meta/table-metadata :users)))
-(deftest database-tables-for-context-prioritization
- (let [used [{:id 1 :name "used1"} {:id 2 :name "used2"}]
- extra [{:id 3 :name "extra1"} {:id 4 :name "extra2"}]
- all-tables (concat used extra)]
- (mt/with-dynamic-fn-redefs [table-utils/used-tables (fn [_] used)
- table-utils/enhanced-database-tables (fn [_ {:keys [priority-tables all-tables-limit] :or {all-tables-limit 100}}]
- (let [priority-ids (set (map :id priority-tables))
- non-priority (remove #(priority-ids (:id %)) all-tables)
- result (concat priority-tables (take (- all-tables-limit (count priority-tables)) non-priority))]
- (take all-tables-limit result)))]
- (let [result (#'context/database-tables-for-context {:query users-native-query, :all-tables-limit 3})]
- (is (= used result) "Should only return used tables, in order")
- (is (= (count used) (count result)) "Should return all used tables")
- (is (not-any? #(= 3 (:id %)) result) "Should not include extra tables")
- (is (not-any? #(= 4 (:id %)) result) "Should not include extra tables")
- (is (= (count (distinct (map :id result))) (count result))))))) ; no duplicates
-
-(deftest database-tables-for-context-used-tables-exceed-limit
- (let [used (mapv #(hash-map :id % :name (str "used" %)) (range 1 6)) ; 5 used tables
- extra (mapv #(hash-map :id % :name (str "extra" %)) (range 6 10))
- all-tables (concat used extra)]
- (mt/with-dynamic-fn-redefs [table-utils/used-tables (fn [_] used)
- table-utils/enhanced-database-tables (fn [_ {:keys [priority-tables all-tables-limit] :or {all-tables-limit 100}}]
- (let [priority-ids (set (map :id priority-tables))
- non-priority (remove #(priority-ids (:id %)) all-tables)
- result (concat priority-tables (take (- all-tables-limit (count priority-tables)) non-priority))]
- (take all-tables-limit result)))]
- (let [result (#'context/database-tables-for-context {:query users-native-query, :all-tables-limit 3})]
- (is (= used result) "Should return all used tables")
- (is (= (count used) (count result)) "Should return all used tables")
- (is (= (count (distinct (map :id result))) (count result))))))) ; no duplicates
+(deftest database-tables-for-context-returns-stubs
+ (testing "Used tables become lightweight stubs — id/type/name/schema/description only, never columns"
+ (let [used [{:id 1 :name "used1" :schema "public" :description "first"}
+ {:id 2 :name "used2" :schema nil :description nil}]]
+ (mt/with-dynamic-fn-redefs [table-utils/used-tables (fn [_] used)]
+ (let [result (#'context/database-tables-for-context {:query users-native-query})]
+ (is (= [{:id 1 :type :table :name "used1" :database_schema "public" :description "first"}
+ {:id 2 :type :table :name "used2" :database_schema nil :description nil}]
+ result)
+ "Should return one stub per used table, in order")
+ (is (not-any? #(contains? % :fields) result)
+ "Stubs must not carry columns — the renderer re-fetches details by id"))))))
(deftest database-tables-for-context-no-used-tables
- (let [extra (mapv #(hash-map :id % :name (str "extra" %)) (range 1 5))]
- (mt/with-dynamic-fn-redefs [table-utils/used-tables (fn [_] [])
- table-utils/enhanced-database-tables (fn [_ opts]
- (take (:all-tables-limit opts 100) extra))]
- (let [result (#'context/database-tables-for-context {:query users-native-query, :all-tables-limit 3})]
- (is (empty? result) "Should return empty when no used tables")))))
-
-(deftest database-tables-for-context-used-tables-exact-limit
- (let [used (mapv #(hash-map :id % :name (str "used" %)) (range 1 4)) ; 3 used tables
- extra (mapv #(hash-map :id % :name (str "extra" %)) (range 4 7))
- all-tables (concat used extra)]
- (mt/with-dynamic-fn-redefs [table-utils/used-tables (fn [_] used)
- table-utils/enhanced-database-tables (fn [_ {:keys [priority-tables all-tables-limit] :or {all-tables-limit 100}}]
- (let [priority-ids (set (map :id priority-tables))
- non-priority (remove #(priority-ids (:id %)) all-tables)
- result (concat priority-tables (take (- all-tables-limit (count priority-tables)) non-priority))]
- (take all-tables-limit result)))]
- (let [result (#'context/database-tables-for-context {:query users-native-query, :all-tables-limit 3})]
- (is (= used result) "Should be exactly the used tables, in order")
- (is (= (count used) (count result)))
- (is (= (count (distinct (map :id result))) (count result))))))) ; no duplicates
+ (mt/with-dynamic-fn-redefs [table-utils/used-tables (fn [_] [])]
+ (let [result (#'context/database-tables-for-context {:query users-native-query})]
+ (is (empty? result) "Should return empty when no used tables"))))
(deftest database-tables-for-context-exception-handling
- (testing "Returns empty when table-utils/enhanced-database-tables throws"
- (mt/with-dynamic-fn-redefs [table-utils/used-tables (fn [_] [{:id 1}])
- table-utils/enhanced-database-tables (fn [_ _] (throw (Exception. "boom")))]
+ (testing "Returns empty when table-utils/used-tables throws"
+ (mt/with-dynamic-fn-redefs [table-utils/used-tables (fn [_] (throw (Exception. "boom")))]
(let [result (#'context/database-tables-for-context {:query users-native-query})]
(is (empty? result))))))
(deftest database-tables-for-context-nil-used-tables
(testing "Handles nil used-tables gracefully"
- (mt/with-dynamic-fn-redefs [table-utils/used-tables (fn [_] nil)
- table-utils/enhanced-database-tables (fn [_ opts]
- (take (:all-tables-limit opts 100) [{:id 1} {:id 2}]))]
+ (mt/with-dynamic-fn-redefs [table-utils/used-tables (fn [_] nil)]
(let [result (#'context/database-tables-for-context {:query users-native-query})]
(is (empty? result) "Should return empty when used-tables is nil")))))
(deftest database-tables-for-context-duplicate-ids
- (testing "Deduplicates tables returned by database-tables"
- (let [used [{:id 1}]
- dup [{:id 1} {:id 1} {:id 2}]]
- (mt/with-dynamic-fn-redefs [table-utils/used-tables (fn [_] used)
- table-utils/enhanced-database-tables (fn [_ _] dup)]
+ (testing "Deduplicates used tables by id"
+ (let [used [{:id 1 :name "a"} {:id 1 :name "a"} {:id 2 :name "b"}]]
+ (mt/with-dynamic-fn-redefs [table-utils/used-tables (fn [_] used)]
(let [result (#'context/database-tables-for-context {:query users-native-query})]
(is (= [1 2] (map :id result)) "Should preserve order and remove duplicates"))))))
@@ -153,6 +111,15 @@
used-tables (get-in result [:user_is_viewing 0 :used_tables])]
(is (= mock-tables used-tables)))))))
+(deftest python-transform-tables-for-context-returns-stubs
+ (testing "Python transform used tables are lightweight stubs without columns"
+ (mt/with-dynamic-fn-redefs [table-utils/used-tables-from-ids
+ (fn [_ _] [{:id 24 :name "orders" :schema "public" :description "orders table"}])]
+ (let [result (#'context/python-transform-tables-for-context {:database-id 2 :table-ids [24]})]
+ (is (= [{:id 24 :type :table :name "orders" :database_schema "public" :description "orders table"}]
+ result))
+ (is (not-any? #(contains? % :fields) result))))))
+
(deftest enhance-context-with-schema-python-transform-no-source-tables
(testing "Handles Python transform without source-tables"
(let [called? (atom false)]
@@ -381,7 +348,9 @@
tables (get-in result [:user_is_viewing 0 :used_tables])]
(is (seq tables) "Should have used_tables for MBQL query")
(is (some #(= table-id (:id %)) tables)
- (str "Should include source table " table-id " in used_tables"))))))
+ (str "Should include source table " table-id " in used_tables"))
+ (is (not-any? #(contains? % :fields) tables)
+ "used_tables are lightweight stubs — columns are fetched at render time by entity-details")))))
(deftest enhance-context-mbql-viewing-context-rendering-test
(testing "MBQL adhoc query viewing context includes table name for LLM"
diff --git a/test/metabase/metabot/table_utils_test.clj b/test/metabase/metabot/table_utils_test.clj
index 307873c16d4..53d490a753e 100644
--- a/test/metabase/metabot/table_utils_test.clj
+++ b/test/metabase/metabot/table_utils_test.clj
@@ -215,23 +215,23 @@
:model/Table {other-db-table-id :id} {:db_id other-db-id, :name "other_table", :schema "public", :active true, :visibility_type nil}]
(testing "returns tables with correct structure for valid table-ids"
(mt/with-current-user (mt/user->id :crowberto)
- (is (= #{{:id table1-id :name "users" :schema "public"}
- {:id table3-id :name "products" :schema "inventory"}}
+ (is (= #{{:id table1-id :name "users" :schema "public" :description nil}
+ {:id table3-id :name "products" :schema "inventory" :description nil}}
(set (table-utils/used-tables-from-ids db-id [table1-id table3-id]))))))
(testing "handles empty table-ids collection"
(mt/with-current-user (mt/user->id :crowberto)
(is (empty? (table-utils/used-tables-from-ids db-id [])))))
(testing "filters by database-id"
(mt/with-current-user (mt/user->id :crowberto)
- (is (= [{:id table1-id :name "users" :schema "public"}]
+ (is (= [{:id table1-id :name "users" :schema "public" :description nil}]
(table-utils/used-tables-from-ids db-id [table1-id other-db-table-id])))))
(testing "filters out inactive tables"
(mt/with-current-user (mt/user->id :crowberto)
- (is (= [{:id table1-id :name "users" :schema "public"}]
+ (is (= [{:id table1-id :name "users" :schema "public" :description nil}]
(table-utils/used-tables-from-ids db-id [table1-id inactive-id])))))
(testing "filters out hidden tables"
(mt/with-current-user (mt/user->id :crowberto)
- (is (= [{:id table1-id :name "users" :schema "public"}]
+ (is (= [{:id table1-id :name "users" :schema "public" :description nil}]
(table-utils/used-tables-from-ids db-id [table1-id hidden-id])))))
(testing "handles non-existent table-ids"
(mt/with-current-user (mt/user->id :crowberto)
@@ -240,8 +240,8 @@
(testing "handles mix of valid and invalid table-ids"
(mt/with-current-user (mt/user->id :crowberto)
(let [fake-id 999999]
- (is (= #{{:id table1-id :name "users" :schema "public"}
- {:id table2-id :name "orders" :schema "public"}}
+ (is (= #{{:id table1-id :name "users" :schema "public" :description nil}
+ {:id table2-id :name "orders" :schema "public" :description nil}}
(set (table-utils/used-tables-from-ids db-id [table1-id fake-id table2-id]))))))))))
;; ======================================
@@ -294,63 +294,6 @@
(is (every? #(not= "hidden_table" (:name %)) tables))
(is (some #(= "visible_table" (:name %)) tables)))))))
-(deftest enhanced-database-tables-test
- (testing "enhanced-database-tables function with new format"
- (mt/with-temp [:model/Database {db-id :id} {}
- :model/Table {table1-id :id} {:db_id db-id, :name "users", :schema "public", :active true, :visibility_type nil}
- :model/Table {table2-id :id} {:db_id db-id, :name "orders", :schema "public", :active true, :visibility_type nil}
- :model/Field {user-id-field :id} {:table_id table1-id, :name "id", :database_type "INTEGER", :base_type :type/Integer, :semantic_type :type/PK}
- :model/Field {} {:table_id table1-id, :name "name", :database_type "VARCHAR", :base_type :type/Text}
- :model/Field {} {:table_id table2-id, :name "id", :database_type "INTEGER", :base_type :type/Integer, :semantic_type :type/PK}
- :model/Field {} {:table_id table2-id, :name "user_id", :database_type "INTEGER", :base_type :type/Integer, :semantic_type :type/FK, :fk_target_field_id user-id-field}
- :model/Field {} {:table_id table2-id, :name "total", :database_type "DECIMAL", :base_type :type/Decimal}]
- (testing "returns tables with new enhanced formatting"
- (mt/with-current-user (mt/user->id :crowberto)
- (let [tables (table-utils/enhanced-database-tables db-id)]
- (is (vector? tables))
- (is (every? #(contains? % :name) tables))
- (is (every? #(contains? % :database_schema) tables))
- (is (every? #(contains? % :fields) tables))
- (is (every? #(contains? % :type) tables))
- (is (every? #(contains? % :display_name) tables))
- (is (every? #(= :table (:type %)) tables))
- (is (every? #(every? (fn [field] (contains? field :field_id)) (:fields %)) tables))
- (is (every? #(every? (fn [field] (contains? field :name)) (:fields %)) tables))
- (is (every? #(every? (fn [field] (contains? field :base_type)) (:fields %)) tables))
- (is (every? #(every? (fn [field] (contains? field :database_type)) (:fields %)) tables))
- (is (every? #(contains? % :metrics) tables)))))
- (testing "includes table_reference for implicitly joined fields"
- (mt/dataset test-data
- (mt/with-current-user (mt/user->id :crowberto)
- (let [test-db-id (mt/id)
- tables (table-utils/enhanced-database-tables test-db-id)
- orders-table (first (filter #(= "ORDERS" (:name %)) tables))
- all-fields (:fields orders-table)
- user-fields (filter #(= "User" (:table_reference %)) all-fields)
- product-fields (filter #(= "Product" (:table_reference %)) all-fields)]
- (is (some? orders-table) "Expected to find ORDERS table")
- (is (seq user-fields) "Expected to find fields with table-reference 'User' from implicit join")
- (is (some #(= "NAME" (:name %)) user-fields) "Expected to find User NAME field from implicit join")
- (is (seq product-fields) "Expected to find fields with table-reference 'Product' from implicit join")
- (is (some #(= "TITLE" (:name %)) product-fields) "Expected to find Product TITLE field from implicit join")))))
- (testing "enhanced format respects all-tables-limit option"
- (mt/with-current-user (mt/user->id :crowberto)
- (let [tables (table-utils/enhanced-database-tables db-id {:all-tables-limit 1})]
- (is (<= (count tables) 1)))))
- (testing "enhanced format excludes specified table IDs"
- (mt/with-current-user (mt/user->id :crowberto)
- (let [all-tables (table-utils/enhanced-database-tables db-id)
- filtered-tables (table-utils/enhanced-database-tables db-id {:exclude-table-ids #{table1-id}})]
- (is (< (count filtered-tables) (count all-tables)))
- (is (not-any? #(= table1-id (:id %)) filtered-tables)))))
- (testing "enhanced format prioritizes specified tables"
- (mt/with-current-user (mt/user->id :crowberto)
- (let [priority-table {:id table2-id :name "orders" :schema "public"}
- tables (table-utils/enhanced-database-tables db-id {:priority-tables [priority-table]})]
- (is (seq tables))
- ;; Priority table should appear first (if it appears at all)
- (is (= "orders" (-> tables first :name)))))))))
-
(deftest ^:parallel format-escaped-test
(are [in out] (= out
(with-out-str (#'table-utils/format-escaped in *out*)))