The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Remove enhanced-database-tables and provide columnless tables in metabot context :used_tables (#76944)
appleby16eefb159d69
diff --git a/src/metabase/metabot/context.clj b/src/metabase/metabot/context.cljindex 7dc10f6776e..bb6bcd975dc 100644--- a/src/metabase/metabot/context.clj+++ b/src/metabase/metabot/context.clj@@ -123,17 +123,30 @@(when (lib/native-only-query? normalized-query)normalized-query)))))+(defn- table-stub+ "Reference to a table used by a viewing-context item.++ Excludes columns. [[metabase.metabot.agent.user-context/format-entity]] only reads `:type` and `:id` and calculates+ column details via [[metabase.metabot.tools.entity-details/get-table-details]], so fetching columns here is wasted+ work. On large, highly-connected schemas it can contribute to heap exhaustion (metabase#76493).+ `:name`/`:database_schema`/`:description` are kept for the `format-simple-entity` fallback"+ [{:keys [id name schema description]}]+ {:id id+ :type :table+ :name name+ :database_schema schema+ :description description})+(defn- database-tables-for-context- "Get database tables formatted for metabot context. Only includes tables used in the query, formatted for API output.- Removes duplicate tables by id while preserving first occurrence order."+ "Get database tables formatted for metabot context. Only includes tables used in the query.+ Removes duplicate tables by id while preserving first occurrence order."[{:keys [query]}](try(if query- (let [used-tables (table-utils/used-tables query)- tables (table-utils/enhanced-database-tables (:database query)- {:priority-tables used-tables- :all-tables-limit (count used-tables)})]- (m/distinct-by :id tables))+ (into []+ (comp (m/distinct-by :id)+ (map table-stub))+ (table-utils/used-tables query))[])(catch Exception e(log/error e "Error getting database tables for context")@@ -154,10 +167,7 @@[{:keys [database-id table-ids]}](try(when (and database-id (seq table-ids))- (when-let [tables (not-empty (table-utils/used-tables-from-ids database-id table-ids))]- (table-utils/enhanced-database-tables database-id- {:priority-tables tables- :all-tables-limit (count tables)})))+ (not-empty (mapv table-stub (table-utils/used-tables-from-ids database-id table-ids))))(catch Exception e(log/error e "Error getting Python transform tables for context")[])))@@ -179,7 +189,7 @@[database-id table-ids]))))(defn- mbql-source-tables-for-context- "Get source tables for an MBQL query, formatted for metabot context (with :type, :display_name, :fields, etc.).+ "Get source tables for an MBQL query, formatted for metabot context.Uses a direct table lookup without native-query permission checks. The user is alreadyviewing these tables in the notebook editor, so they have at least query-builder access.@@ -187,16 +197,15 @@which is too restrictive for MBQL viewing context enrichment."[[database-id table-ids]](try- (let [raw-tables (t2/select [:model/Table :id :name :schema]+ (let [raw-tables (t2/select [:model/Table :id :name :schema :description]:db_id database-id:id [:in table-ids]:active true- :visibility_type nil)- tables (when (seq raw-tables)- (table-utils/enhanced-database-tables database-id- {:priority-tables raw-tables- :all-tables-limit (count raw-tables)}))]- (m/distinct-by :id tables))+ :visibility_type nil)]+ (into []+ (comp (m/distinct-by :id)+ (map table-stub))+ raw-tables))(catch Exception e(log/error e "Error getting MBQL source tables for context")nil)))diff --git a/src/metabase/metabot/table_utils.clj b/src/metabase/metabot/table_utils.cljindex db780537712..49442d539fb 100644--- a/src/metabase/metabot/table_utils.clj+++ b/src/metabase/metabot/table_utils.clj@@ -3,14 +3,9 @@(:require[clojure.set :as set][metabase.api.common :as api]- [metabase.lib-be.core :as lib-be]- [metabase.lib.core :as lib]- [metabase.lib.metadata :as lib.metadata][metabase.metabot.query-analyzer :as query-analyzer]- [metabase.metabot.tools.util :as metabot.tools.u][metabase.models.interface :as mi][metabase.util :as u]- [metabase.util.humanization :as u.humanization][toucan2.core :as t2][toucan2.realize :as t2.realize])(:import@@ -73,58 +68,6 @@fields)}))all-tables))))-(defn enhanced-database-tables- "Get database tables formatted with the new metabot tools schema format.-- Returns tables with :type, :display_name, :database_id, :database_schema, :fields (with field-id), :metrics.- This format is used by metabot context and other modern tools."- ([database-id]- (enhanced-database-tables database-id nil))- ([database-id {:keys [all-tables-limit priority-tables exclude-table-ids]- :or {all-tables-limit max-database-tables- priority-tables []- exclude-table-ids #{}}}]- (let [priority-table-ids (set (map :id priority-tables))- {table-where-clause :clause table-cte :with} (mi/visible-filter-clause :model/Table- :id- {:user-id api/*current-user-id*- :is-superuser? api/*is-superuser?*}- {:perms/view-data :unrestricted- :perms/create-queries :query-builder-and-native})- ;; Fetch most viewed tables, excluding priority tables and excluded tables- fill-tables (t2/select [:model/Table :id :db_id :name :schema :description]- :db_id database-id- :active true- :visibility_type nil- (cond-> {:where table-where-clause- :order-by [[:view_count :desc]]- :limit all-tables-limit}- table-cte (assoc :with table-cte)))- fill-tables (remove #(or (priority-table-ids (:id %))- (exclude-table-ids (:id %))) fill-tables)- all-tables (concat priority-tables fill-tables)- all-tables (take all-tables-limit all-tables)]- (lib-be/with-metadata-provider-cache- (let [mp (lib-be/application-database-metadata-provider database-id)- table-ids (map :id all-tables)- _ (lib.metadata/bulk-metadata mp :metadata/table table-ids)- engine (:engine (lib.metadata/database mp))]- (mapv (fn [{:keys [id name schema description]}]- (let [table-query (lib/query mp (lib.metadata/table mp id))- cols (->> (lib/visible-columns table-query)- (map #(metabot.tools.u/add-table-reference table-query %)))]- {:id id- :type :table- :name name- :display_name (u.humanization/name->human-readable-name :simple name)- :database_id database-id- :database_engine engine- :database_schema schema- :description description- :fields (mapv #(metabot.tools.u/->result-column table-query %) cols)- :metrics []}))- all-tables))))))-(defn get-tables"Get information about the tables in a given database.@@ -198,7 +141,7 @@(keep (fn [table](when (some #(matching-tables? table % {:match-schema? false}) unrecognized-tables)(t2.realize/realize table))))- (t2/reducible-select [:model/Table :id :name :schema]+ (t2/reducible-select [:model/Table :id :name :schema :description]:db_id database-id:active true:visibility_type nil@@ -215,7 +158,7 @@[database-id table-ids](if-not (seq table-ids)[]- (t2/select [:model/Table :id :name :schema]+ (t2/select [:model/Table :id :name :schema :description]:db_id database-id:id [:in table-ids]:active truediff --git a/test/metabase/metabot/context_test.clj b/test/metabase/metabot/context_test.cljindex 833a496ad1a..516cbe585d4 100644--- a/test/metabase/metabot/context_test.clj+++ b/test/metabase/metabot/context_test.clj@@ -16,82 +16,40 @@(def ^:private users-native-query (lib/native-query meta/metadata-provider "SELECT * FROM users"))(def ^:private users-mbql-query (lib/query meta/metadata-provider (meta/table-metadata :users)))-(deftest database-tables-for-context-prioritization- (let [used [{:id 1 :name "used1"} {:id 2 :name "used2"}]- extra [{:id 3 :name "extra1"} {:id 4 :name "extra2"}]- all-tables (concat used extra)]- (mt/with-dynamic-fn-redefs [table-utils/used-tables (fn [_] used)- table-utils/enhanced-database-tables (fn [_ {:keys [priority-tables all-tables-limit] :or {all-tables-limit 100}}]- (let [priority-ids (set (map :id priority-tables))- non-priority (remove #(priority-ids (:id %)) all-tables)- result (concat priority-tables (take (- all-tables-limit (count priority-tables)) non-priority))]- (take all-tables-limit result)))]- (let [result (#'context/database-tables-for-context {:query users-native-query, :all-tables-limit 3})]- (is (= used result) "Should only return used tables, in order")- (is (= (count used) (count result)) "Should return all used tables")- (is (not-any? #(= 3 (:id %)) result) "Should not include extra tables")- (is (not-any? #(= 4 (:id %)) result) "Should not include extra tables")- (is (= (count (distinct (map :id result))) (count result))))))) ; no duplicates--(deftest database-tables-for-context-used-tables-exceed-limit- (let [used (mapv #(hash-map :id % :name (str "used" %)) (range 1 6)) ; 5 used tables- extra (mapv #(hash-map :id % :name (str "extra" %)) (range 6 10))- all-tables (concat used extra)]- (mt/with-dynamic-fn-redefs [table-utils/used-tables (fn [_] used)- table-utils/enhanced-database-tables (fn [_ {:keys [priority-tables all-tables-limit] :or {all-tables-limit 100}}]- (let [priority-ids (set (map :id priority-tables))- non-priority (remove #(priority-ids (:id %)) all-tables)- result (concat priority-tables (take (- all-tables-limit (count priority-tables)) non-priority))]- (take all-tables-limit result)))]- (let [result (#'context/database-tables-for-context {:query users-native-query, :all-tables-limit 3})]- (is (= used result) "Should return all used tables")- (is (= (count used) (count result)) "Should return all used tables")- (is (= (count (distinct (map :id result))) (count result))))))) ; no duplicates+(deftest database-tables-for-context-returns-stubs+ (testing "Used tables become lightweight stubs — id/type/name/schema/description only, never columns"+ (let [used [{:id 1 :name "used1" :schema "public" :description "first"}+ {:id 2 :name "used2" :schema nil :description nil}]]+ (mt/with-dynamic-fn-redefs [table-utils/used-tables (fn [_] used)]+ (let [result (#'context/database-tables-for-context {:query users-native-query})]+ (is (= [{:id 1 :type :table :name "used1" :database_schema "public" :description "first"}+ {:id 2 :type :table :name "used2" :database_schema nil :description nil}]+ result)+ "Should return one stub per used table, in order")+ (is (not-any? #(contains? % :fields) result)+ "Stubs must not carry columns — the renderer re-fetches details by id"))))))(deftest database-tables-for-context-no-used-tables- (let [extra (mapv #(hash-map :id % :name (str "extra" %)) (range 1 5))]- (mt/with-dynamic-fn-redefs [table-utils/used-tables (fn [_] [])- table-utils/enhanced-database-tables (fn [_ opts]- (take (:all-tables-limit opts 100) extra))]- (let [result (#'context/database-tables-for-context {:query users-native-query, :all-tables-limit 3})]- (is (empty? result) "Should return empty when no used tables")))))--(deftest database-tables-for-context-used-tables-exact-limit- (let [used (mapv #(hash-map :id % :name (str "used" %)) (range 1 4)) ; 3 used tables- extra (mapv #(hash-map :id % :name (str "extra" %)) (range 4 7))- all-tables (concat used extra)]- (mt/with-dynamic-fn-redefs [table-utils/used-tables (fn [_] used)- table-utils/enhanced-database-tables (fn [_ {:keys [priority-tables all-tables-limit] :or {all-tables-limit 100}}]- (let [priority-ids (set (map :id priority-tables))- non-priority (remove #(priority-ids (:id %)) all-tables)- result (concat priority-tables (take (- all-tables-limit (count priority-tables)) non-priority))]- (take all-tables-limit result)))]- (let [result (#'context/database-tables-for-context {:query users-native-query, :all-tables-limit 3})]- (is (= used result) "Should be exactly the used tables, in order")- (is (= (count used) (count result)))- (is (= (count (distinct (map :id result))) (count result))))))) ; no duplicates+ (mt/with-dynamic-fn-redefs [table-utils/used-tables (fn [_] [])]+ (let [result (#'context/database-tables-for-context {:query users-native-query})]+ (is (empty? result) "Should return empty when no used tables"))))(deftest database-tables-for-context-exception-handling- (testing "Returns empty when table-utils/enhanced-database-tables throws"- (mt/with-dynamic-fn-redefs [table-utils/used-tables (fn [_] [{:id 1}])- table-utils/enhanced-database-tables (fn [_ _] (throw (Exception. "boom")))]+ (testing "Returns empty when table-utils/used-tables throws"+ (mt/with-dynamic-fn-redefs [table-utils/used-tables (fn [_] (throw (Exception. "boom")))](let [result (#'context/database-tables-for-context {:query users-native-query})](is (empty? result))))))(deftest database-tables-for-context-nil-used-tables(testing "Handles nil used-tables gracefully"- (mt/with-dynamic-fn-redefs [table-utils/used-tables (fn [_] nil)- table-utils/enhanced-database-tables (fn [_ opts]- (take (:all-tables-limit opts 100) [{:id 1} {:id 2}]))]+ (mt/with-dynamic-fn-redefs [table-utils/used-tables (fn [_] nil)](let [result (#'context/database-tables-for-context {:query users-native-query})](is (empty? result) "Should return empty when used-tables is nil")))))(deftest database-tables-for-context-duplicate-ids- (testing "Deduplicates tables returned by database-tables"- (let [used [{:id 1}]- dup [{:id 1} {:id 1} {:id 2}]]- (mt/with-dynamic-fn-redefs [table-utils/used-tables (fn [_] used)- table-utils/enhanced-database-tables (fn [_ _] dup)]+ (testing "Deduplicates used tables by id"+ (let [used [{:id 1 :name "a"} {:id 1 :name "a"} {:id 2 :name "b"}]]+ (mt/with-dynamic-fn-redefs [table-utils/used-tables (fn [_] used)](let [result (#'context/database-tables-for-context {:query users-native-query})](is (= [1 2] (map :id result)) "Should preserve order and remove duplicates"))))))@@ -153,6 +111,15 @@used-tables (get-in result [:user_is_viewing 0 :used_tables])](is (= mock-tables used-tables)))))))+(deftest python-transform-tables-for-context-returns-stubs+ (testing "Python transform used tables are lightweight stubs without columns"+ (mt/with-dynamic-fn-redefs [table-utils/used-tables-from-ids+ (fn [_ _] [{:id 24 :name "orders" :schema "public" :description "orders table"}])]+ (let [result (#'context/python-transform-tables-for-context {:database-id 2 :table-ids [24]})]+ (is (= [{:id 24 :type :table :name "orders" :database_schema "public" :description "orders table"}]+ result))+ (is (not-any? #(contains? % :fields) result))))))+(deftest enhance-context-with-schema-python-transform-no-source-tables(testing "Handles Python transform without source-tables"(let [called? (atom false)]@@ -381,7 +348,9 @@tables (get-in result [:user_is_viewing 0 :used_tables])](is (seq tables) "Should have used_tables for MBQL query")(is (some #(= table-id (:id %)) tables)- (str "Should include source table " table-id " in used_tables"))))))+ (str "Should include source table " table-id " in used_tables"))+ (is (not-any? #(contains? % :fields) tables)+ "used_tables are lightweight stubs — columns are fetched at render time by entity-details")))))(deftest enhance-context-mbql-viewing-context-rendering-test(testing "MBQL adhoc query viewing context includes table name for LLM"diff --git a/test/metabase/metabot/table_utils_test.clj b/test/metabase/metabot/table_utils_test.cljindex 307873c16d4..53d490a753e 100644--- a/test/metabase/metabot/table_utils_test.clj+++ b/test/metabase/metabot/table_utils_test.clj@@ -215,23 +215,23 @@:model/Table {other-db-table-id :id} {:db_id other-db-id, :name "other_table", :schema "public", :active true, :visibility_type nil}](testing "returns tables with correct structure for valid table-ids"(mt/with-current-user (mt/user->id :crowberto)- (is (= #{{:id table1-id :name "users" :schema "public"}- {:id table3-id :name "products" :schema "inventory"}}+ (is (= #{{:id table1-id :name "users" :schema "public" :description nil}+ {:id table3-id :name "products" :schema "inventory" :description nil}}(set (table-utils/used-tables-from-ids db-id [table1-id table3-id]))))))(testing "handles empty table-ids collection"(mt/with-current-user (mt/user->id :crowberto)(is (empty? (table-utils/used-tables-from-ids db-id [])))))(testing "filters by database-id"(mt/with-current-user (mt/user->id :crowberto)- (is (= [{:id table1-id :name "users" :schema "public"}]+ (is (= [{:id table1-id :name "users" :schema "public" :description nil}](table-utils/used-tables-from-ids db-id [table1-id other-db-table-id])))))(testing "filters out inactive tables"(mt/with-current-user (mt/user->id :crowberto)- (is (= [{:id table1-id :name "users" :schema "public"}]+ (is (= [{:id table1-id :name "users" :schema "public" :description nil}](table-utils/used-tables-from-ids db-id [table1-id inactive-id])))))(testing "filters out hidden tables"(mt/with-current-user (mt/user->id :crowberto)- (is (= [{:id table1-id :name "users" :schema "public"}]+ (is (= [{:id table1-id :name "users" :schema "public" :description nil}](table-utils/used-tables-from-ids db-id [table1-id hidden-id])))))(testing "handles non-existent table-ids"(mt/with-current-user (mt/user->id :crowberto)@@ -240,8 +240,8 @@(testing "handles mix of valid and invalid table-ids"(mt/with-current-user (mt/user->id :crowberto)(let [fake-id 999999]- (is (= #{{:id table1-id :name "users" :schema "public"}- {:id table2-id :name "orders" :schema "public"}}+ (is (= #{{:id table1-id :name "users" :schema "public" :description nil}+ {:id table2-id :name "orders" :schema "public" :description nil}}(set (table-utils/used-tables-from-ids db-id [table1-id fake-id table2-id]))))))))));; ======================================@@ -294,63 +294,6 @@(is (every? #(not= "hidden_table" (:name %)) tables))(is (some #(= "visible_table" (:name %)) tables)))))))-(deftest enhanced-database-tables-test- (testing "enhanced-database-tables function with new format"- (mt/with-temp [:model/Database {db-id :id} {}- :model/Table {table1-id :id} {:db_id db-id, :name "users", :schema "public", :active true, :visibility_type nil}- :model/Table {table2-id :id} {:db_id db-id, :name "orders", :schema "public", :active true, :visibility_type nil}- :model/Field {user-id-field :id} {:table_id table1-id, :name "id", :database_type "INTEGER", :base_type :type/Integer, :semantic_type :type/PK}- :model/Field {} {:table_id table1-id, :name "name", :database_type "VARCHAR", :base_type :type/Text}- :model/Field {} {:table_id table2-id, :name "id", :database_type "INTEGER", :base_type :type/Integer, :semantic_type :type/PK}- :model/Field {} {:table_id table2-id, :name "user_id", :database_type "INTEGER", :base_type :type/Integer, :semantic_type :type/FK, :fk_target_field_id user-id-field}- :model/Field {} {:table_id table2-id, :name "total", :database_type "DECIMAL", :base_type :type/Decimal}]- (testing "returns tables with new enhanced formatting"- (mt/with-current-user (mt/user->id :crowberto)- (let [tables (table-utils/enhanced-database-tables db-id)]- (is (vector? tables))- (is (every? #(contains? % :name) tables))- (is (every? #(contains? % :database_schema) tables))- (is (every? #(contains? % :fields) tables))- (is (every? #(contains? % :type) tables))- (is (every? #(contains? % :display_name) tables))- (is (every? #(= :table (:type %)) tables))- (is (every? #(every? (fn [field] (contains? field :field_id)) (:fields %)) tables))- (is (every? #(every? (fn [field] (contains? field :name)) (:fields %)) tables))- (is (every? #(every? (fn [field] (contains? field :base_type)) (:fields %)) tables))- (is (every? #(every? (fn [field] (contains? field :database_type)) (:fields %)) tables))- (is (every? #(contains? % :metrics) tables)))))- (testing "includes table_reference for implicitly joined fields"- (mt/dataset test-data- (mt/with-current-user (mt/user->id :crowberto)- (let [test-db-id (mt/id)- tables (table-utils/enhanced-database-tables test-db-id)- orders-table (first (filter #(= "ORDERS" (:name %)) tables))- all-fields (:fields orders-table)- user-fields (filter #(= "User" (:table_reference %)) all-fields)- product-fields (filter #(= "Product" (:table_reference %)) all-fields)]- (is (some? orders-table) "Expected to find ORDERS table")- (is (seq user-fields) "Expected to find fields with table-reference 'User' from implicit join")- (is (some #(= "NAME" (:name %)) user-fields) "Expected to find User NAME field from implicit join")- (is (seq product-fields) "Expected to find fields with table-reference 'Product' from implicit join")- (is (some #(= "TITLE" (:name %)) product-fields) "Expected to find Product TITLE field from implicit join")))))- (testing "enhanced format respects all-tables-limit option"- (mt/with-current-user (mt/user->id :crowberto)- (let [tables (table-utils/enhanced-database-tables db-id {:all-tables-limit 1})]- (is (<= (count tables) 1)))))- (testing "enhanced format excludes specified table IDs"- (mt/with-current-user (mt/user->id :crowberto)- (let [all-tables (table-utils/enhanced-database-tables db-id)- filtered-tables (table-utils/enhanced-database-tables db-id {:exclude-table-ids #{table1-id}})]- (is (< (count filtered-tables) (count all-tables)))- (is (not-any? #(= table1-id (:id %)) filtered-tables)))))- (testing "enhanced format prioritizes specified tables"- (mt/with-current-user (mt/user->id :crowberto)- (let [priority-table {:id table2-id :name "orders" :schema "public"}- tables (table-utils/enhanced-database-tables db-id {:priority-tables [priority-table]})]- (is (seq tables))- ;; Priority table should appear first (if it appears at all)- (is (= "orders" (-> tables first :name)))))))))-(deftest ^:parallel format-escaped-test(are [in out] (= out(with-out-str (#'table-utils/format-escaped in *out*)))