Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Fix database tools perf (#72633)

Alexander Polyankin1526a1b26ebf
diff --git a/src/metabase/agent_api/api.clj b/src/metabase/agent_api/api.clj
index 5ea4e9c9750..214869705cd 100644
--- a/src/metabase/agent_api/api.clj
+++ b/src/metabase/agent_api/api.clj
@@ -130,7 +130,7 @@
[:database_engine :string]
[:database_schema {:optional true} [:maybe :string]]
[:description {:optional true} [:maybe :string]]
- [:fields [:sequential ::field]]
+ [:fields {:optional true} [:maybe [:sequential ::field]]]
[:related_tables {:optional true} [:maybe [:sequential ::related-table]]]
[:metrics {:optional true} [:maybe [:sequential ::metric-summary]]]
[:measures {:optional true} [:maybe [:sequential ::measure]]]
@@ -222,27 +222,16 @@
{:scope metabot/agent-database-read
:tool {:name "get_database"}}
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
- {:keys [with-tables with-fields with-field-values with-related-tables with-metrics with-measures with-segments]
- :or {with-tables false, with-fields false, with-field-values false, with-related-tables false,
- with-metrics false, with-measures false, with-segments false}}
+ {:keys [with-tables with-fields]
+ :or {with-tables false, with-fields false}}
:- [:map
- [:with-tables {:optional true} [:maybe :boolean]]
- [:with-fields {:optional true} [:maybe :boolean]]
- [:with-field-values {:optional true} [:maybe :boolean]]
- [:with-related-tables {:optional true} [:maybe :boolean]]
- [:with-metrics {:optional true} [:maybe :boolean]]
- [:with-measures {:optional true} [:maybe :boolean]]
- [:with-segments {:optional true} [:maybe :boolean]]]]
+ [:with-tables {:optional true} [:maybe :boolean]]
+ [:with-fields {:optional true} [:maybe :boolean]]]]
(check-tool-result
(entity-details/get-database-details
- {:database-id id
- :with-tables? with-tables
- :with-fields? with-fields
- :with-field-values? with-field-values
- :with-related-tables? with-related-tables
- :with-metrics? with-metrics
- :with-measures? with-measures
- :with-segments? with-segments})))
+ {:database-id id
+ :with-tables? with-tables
+ :with-fields? with-fields})))
(api.macros/defendpoint :get "/v1/table/:id" :- ::table
"Get details for a table by ID."
diff --git a/src/metabase/metabot/tools/entity_details.clj b/src/metabase/metabot/tools/entity_details.clj
index ca574ef5030..8db9b64c6f8 100644
--- a/src/metabase/metabot/tools/entity_details.clj
+++ b/src/metabase/metabot/tools/entity_details.clj
@@ -192,7 +192,7 @@
db-id (if metadata-provider (:db-id base) (:db_id base))
db-engine (some-> (if metadata-provider
(lib.metadata/database metadata-provider)
- (metabot.tools.u/get-database db-id :engine))
+ (metabot.tools.u/get-database db-id))
:engine name)
mp (when query-needed?
(or metadata-provider
@@ -207,18 +207,9 @@
(metabot.tools.u/table-field-id-prefix id))
related-tables (when with-related-tables?
(related-tables table-query field-id-prefix with-fields? field-values-fn))]
- (-> {:id id
- :type :table
- :fields (into [] (map-indexed #(metabot.tools.u/->result-column table-query %2 %1 field-id-prefix)) cols)
- :name (:name base)
- ;; :display_name should be (lib/display-name table-query), but we want to avoid creating the query if possible
- :display_name (some->> (:name base)
- (u.humanization/name->human-readable-name :simple))
- :database_id db-id
- :database_engine db-engine
- :database_schema (:schema base)}
- (m/assoc-some :description (:description base)
- :related_tables related-tables
+ (-> (metabot.tools.u/->result-table base db-engine)
+ (assoc :fields (into [] (map-indexed #(metabot.tools.u/->result-column table-query %2 %1 field-id-prefix)) cols))
+ (m/assoc-some :related_tables related-tables
:metrics (when with-metrics?
(not-empty (mapv #(convert-metric % mp options)
(lib/available-metrics table-query))))
@@ -394,46 +385,47 @@
{:output (ex-message e) :status-code 404}
(metabot.tools.u/handle-agent-error e)))))
-(defn- database-details
- "Convert a Database model to the agent API response shape."
- [db]
- {:id (:id db)
- :name (:name db)
- :engine (some-> (:engine db) name)})
-
(defn list-databases
"Get a list of all databases the current user can read without including its tables."
[]
(try
- (let [dbs (metabot.tools.u/list-databases :name :engine)]
- {:structured-output (mapv database-details dbs)})
+ (let [dbs (metabot.tools.u/list-databases)]
+ {:structured-output (mapv metabot.tools.u/->result-database dbs)})
(catch Exception e
(metabot.tools.u/handle-agent-error e))))
+(defn- database-table-details
+ "Build a table result map for `get-database-details`, optionally including its fields."
+ [table engine table-id->fields with-fields?]
+ (let [id (:id table)
+ prefix (metabot.tools.u/table-field-id-prefix id)
+ cols (->> (get table-id->fields id)
+ (sort-by (juxt (fn [f] (or (:position f) 0)) :id)))]
+ (cond-> (metabot.tools.u/->result-table table engine)
+ with-fields?
+ (assoc :fields
+ (into []
+ (map-indexed
+ (fn [i field]
+ (metabot.tools.u/->result-column field i prefix)))
+ cols)))))
+
(defn get-database-details
- "Get details for a database by ID, optionally including its tables."
- [{:keys [database-id with-tables? with-fields? with-field-values? with-related-tables?
- with-metrics? with-measures? with-segments?]
- :or {with-tables? false, with-fields? false, with-field-values? false, with-related-tables? false,
- with-metrics? false, with-measures? false, with-segments? false}}]
+ "Get details for a database by ID, optionally including its tables and fields.
+ Uses raw t2 queries so it stays efficient on databases with many tables and fields."
+ [{:keys [database-id with-tables? with-fields?]
+ :or {with-tables? false, with-fields? false}}]
(try
- (lib-be/with-metadata-provider-cache
- (let [db (metabot.tools.u/get-database database-id :name :engine)
- response (database-details db)]
- {:structured-output
- (cond-> response
- with-tables?
- (assoc :tables
- (let [mp (lib-be/application-database-metadata-provider database-id)
- tables (lib.metadata/tables mp)
- opts (cond-> {:metadata-provider mp
- :with-fields? with-fields?
- :with-related-tables? with-related-tables?
- :with-metrics? with-metrics?
- :with-measures? with-measures?
- :with-segments? with-segments?}
- (not with-field-values?) (assoc :field-values-fn identity))]
- (mapv #(table-details (:id %) opts) tables))))}))
+ (let [db (metabot.tools.u/get-database database-id)
+ engine (some-> (:engine db) name)
+ tables (when with-tables?
+ (metabot.tools.u/list-database-tables database-id))
+ table-id->fields (when with-fields?
+ (group-by :table_id (metabot.tools.u/list-database-table-fields database-id)))]
+ {:structured-output
+ (cond-> (metabot.tools.u/->result-database db)
+ with-tables?
+ (assoc :tables (mapv #(database-table-details % engine table-id->fields with-fields?) tables)))})
(catch Exception e
(if (= (:status-code (ex-data e)) 404)
{:output (ex-message e) :status-code 404}
diff --git a/src/metabase/metabot/tools/util.clj b/src/metabase/metabot/tools/util.clj
index cf03039067e..2ef3ef39c46 100644
--- a/src/metabase/metabot/tools/util.clj
+++ b/src/metabase/metabot/tools/util.clj
@@ -11,6 +11,7 @@
[metabase.models.interface :as mi]
[metabase.premium-features.core :as premium-features]
[metabase.util :as u]
+ [metabase.util.humanization :as u.humanization]
[metabase.util.log :as log]
[toucan2.core :as t2]))
@@ -53,27 +54,60 @@
"A prefix pattern accepting columns from any entity."
#"^.*-(\d+)")
+(defn ->result-database
+ "Build the agent result map for a database."
+ [db]
+ {:id (:id db)
+ :name (:name db)
+ :engine (some-> (:engine db) name)})
+
+(defn ->result-table
+ "Build the base agent result map for a table (without `:fields`).
+ Accepts either a raw t2 row (underscore keys) or a lib-shaped table (hyphenated keys)."
+ [table db-engine]
+ (let [table (u/kebab->snake-keys table)]
+ (cond-> {:id (:id table)
+ :type :table
+ :name (:name table)
+ :display_name (or (:display_name table)
+ (some->> (:name table) (u.humanization/name->human-readable-name :simple)))
+ :database_id (:db_id table)
+ :database_engine db-engine
+ :database_schema (:schema table)}
+ (:description table) (assoc :description (:description table)))))
+
(defn ->result-column
- "Return tool result columns for `column` of `query`. The position of `column` is determined by `index`.
- The ID is then generated by prefixing the position with `field-id-prefix`."
- [query column index field-id-prefix]
- (let [base-type (some-> (:base-type column) u/qualified-name)
- effective-type (some-> (:effective-type column) u/qualified-name)
- semantic-type (some-> (:semantic-type column) u/qualified-name)
- coercion-strategy (some-> (:coercion-strategy column) u/qualified-name)]
- (-> {:field_id (str field-id-prefix index)
- ;; Prefer desired-column-alias over source-column-alias to ensure we use deduplicated names if needed
- :name (or (:lib/desired-column-alias column)
- (:lib/source-column-alias column))
- :display_name (lib/display-name query column)}
- (m/assoc-some :description (:description column)
- :base_type base-type
- :effective_type (when (not= effective-type base-type) effective-type)
- :semantic_type semantic-type
- :database_type (:database-type column)
- :coercion_strategy coercion-strategy
- :field_values (:field-values column)
- :table_reference (:table-reference column)))))
+ "Return a tool result column for `column`. Position is determined by `index`; the field ID is
+ `field-id-prefix` + `index`.
+
+ Accepts either a raw t2 row (underscore keys) or a lib-shaped column (hyphenated keys).
+
+ The 3-arity takes the display name directly from the column; the 4-arity computes it via
+ `(lib/display-name query column)`."
+ ([column index field-id-prefix]
+ (let [column (u/kebab->snake-keys column)
+ base-type (some-> (:base_type column) u/qualified-name)
+ effective-type (some-> (:effective_type column) u/qualified-name)
+ semantic-type (some-> (:semantic_type column) u/qualified-name)
+ coercion-strategy (some-> (:coercion_strategy column) u/qualified-name)]
+ (-> {:field_id (str field-id-prefix index)
+ ;; Prefer desired-column-alias over source-column-alias to ensure we use deduplicated names if needed
+ :name (or (:lib/desired_column_alias column)
+ (:lib/source_column_alias column)
+ (:name column))
+ :display_name (:display_name column)}
+ (m/assoc-some :description (:description column)
+ :base_type base-type
+ :effective_type (when (not= effective-type base-type) effective-type)
+ :semantic_type semantic-type
+ :database_type (:database_type column)
+ :coercion_strategy coercion-strategy
+ :field_values (:field_values column)
+ :table_reference (:table_reference column)))))
+ ([query column index field-id-prefix]
+ (->result-column (assoc column :display-name (lib/display-name query column))
+ index
+ field-id-prefix)))
(defn parse-field-id
"Parse a field-id string into its components.
@@ -161,18 +195,58 @@
second))
:schedule_frame (some->> day-of-month name (re-find #"^(?:first|mid|last)"))})
+(defn- database-filter
+ "HoneySQL WHERE clause matching tables (alias `:t`) belonging to the database with `database-id`."
+ [database-id]
+ [:= :t.db_id database-id])
+
+(defn- table-filter
+ "HoneySQL WHERE clause keeping only user-visible tables (alias `:t`): active and not hidden,
+ technical, or cruft."
+ []
+ [:and [:= :t.active true]
+ [:or
+ [:= :t.visibility_type nil]
+ [:not-in :t.visibility_type [:inline ["hidden" "technical" "cruft"]]]]])
+
+(defn- field-filter
+ "HoneySQL WHERE clause keeping only user-visible fields (alias `:f`): active and not retired
+ or sensitive."
+ []
+ [:and [:= :f.active true]
+ [:or
+ [:= :f.visibility_type nil]
+ [:not-in :f.visibility_type [:inline ["retired" "sensitive"]]]]])
+
+(defn list-databases
+ "Get all databases the current user can read."
+ []
+ (->> (t2/select [:model/Database :id :name :engine])
+ (filterv mi/can-read?)))
+
(defn get-database
- "Get the `fields` of the database with ID `id`."
- [id & fields]
- (-> (t2/select-one (into [:model/Database :id] fields) id)
+ "Get the database with ID `database-id`."
+ [database-id]
+ (-> (t2/select-one [:model/Database :id :name :engine] :id database-id)
api/read-check))
-(defn list-databases
- "Get all databases the current user can read, ordered by name."
- [& fields]
- (->> (t2/select (into [:model/Database :id] fields) {:order-by [[:name :asc]]})
+(defn list-database-tables
+ "Get all active tables in the database with ID `database-id`."
+ [database-id]
+ (->> (t2/select [:model/Table :t.id :t.db_id :t.name :t.schema :t.display_name :t.description]
+ {:from [[(t2/table-name :model/Table) :t]]
+ :where [:and (database-filter database-id) (table-filter)]})
(filterv mi/can-read?)))
+(defn list-database-table-fields
+ "Get all active fields for active tables in the database with ID `database-id`."
+ [database-id]
+ (t2/select [:model/Field :f.id :f.table_id :f.position :f.name :f.display_name :f.description
+ :f.base_type :f.effective_type :f.semantic_type :f.coercion_strategy :f.database_type]
+ {:from [[(t2/table-name :model/Field) :f]]
+ :join [[(t2/table-name :model/Table) :t] [:= :t.id :f.table_id]]
+ :where [:and (database-filter database-id) (table-filter) (field-filter)]}))
+
(defn get-table
"Get the `fields` of the table with ID `id`."
[id & fields]
diff --git a/test/metabase/agent_api/api_test.clj b/test/metabase/agent_api/api_test.clj
index d3886196c5e..81ef3e1895d 100644
--- a/test/metabase/agent_api/api_test.clj
+++ b/test/metabase/agent_api/api_test.clj
@@ -122,33 +122,42 @@
(deftest get-table-details-field-types-test
(testing "Field metadata (base_type, effective_type, semantic_type, coercion_strategy) is returned correctly"
- (mt/with-temp [:model/Database {db-id :id} {}
- :model/Table {table-id :id} {:db_id db-id, :name "t", :active true}
- :model/Field _ {:table_id table-id, :name "id"
- :base_type :type/BigInteger
- :semantic_type :type/PK}
- :model/Field _ {:table_id table-id, :name "name"
- :base_type :type/Text}
- :model/Field _ {:table_id table-id, :name "created_at"
- :base_type :type/Text
- :effective_type :type/DateTime
- :coercion_strategy :Coercion/ISO8601->DateTime}]
- (let [fields (-> (mt/user-http-request :rasta :get 200 (str "agent/v1/table/" table-id))
- :fields)
- by-name (m/index-by :name fields)]
- (testing "base_type is always set"
- (is (= "type/BigInteger" (get-in by-name ["id" :base_type])))
- (is (= "type/Text" (get-in by-name ["name" :base_type])))
- (is (= "type/Text" (get-in by-name ["created_at" :base_type]))))
- (testing "semantic_type is returned when set"
- (is (= "type/PK" (get-in by-name ["id" :semantic_type])))
- (is (nil? (get-in by-name ["name" :semantic_type]))))
- (testing "effective_type and coercion_strategy are returned when coerced"
- (is (= "type/DateTime" (get-in by-name ["created_at" :effective_type])))
- (is (= "Coercion/ISO8601->DateTime" (get-in by-name ["created_at" :coercion_strategy]))))
- (testing "effective_type is omitted when it equals base_type"
- (is (not (contains? (get by-name "id") :effective_type)))
- (is (not (contains? (get by-name "name") :effective_type))))))))
+ (testing "base_type is always set, semantic_type is returned when set"
+ (mt/with-temp [:model/Database {db-id :id} {}
+ :model/Table {table-id :id} {:db_id db-id, :name "t", :active true}
+ :model/Field _ {:table_id table-id, :name "id"
+ :base_type :type/BigInteger
+ :semantic_type :type/PK}]
+ (is (=? {:fields [{:name "id"
+ :base_type "type/BigInteger"
+ :semantic_type "type/PK"}]}
+ (mt/user-http-request :rasta :get 200 (str "agent/v1/table/" table-id))))))
+
+ (testing "semantic_type is omitted when not set, effective_type is omitted when it equals base_type"
+ (mt/with-temp [:model/Database {db-id :id} {}
+ :model/Table {table-id :id} {:db_id db-id, :name "t", :active true}
+ :model/Field _ {:table_id table-id, :name "name"
+ :base_type :type/Text}]
+ (let [[field :as fields] (:fields (mt/user-http-request :rasta :get 200 (str "agent/v1/table/" table-id)))]
+ (is (= 1 (count fields)))
+ (is (=? {:name "name"
+ :base_type "type/Text"}
+ field))
+ (is (not (contains? field :semantic_type)))
+ (is (not (contains? field :effective_type))))))
+
+ (testing "effective_type and coercion_strategy are returned when coerced"
+ (mt/with-temp [:model/Database {db-id :id} {}
+ :model/Table {table-id :id} {:db_id db-id, :name "t", :active true}
+ :model/Field _ {:table_id table-id, :name "created_at"
+ :base_type :type/Text
+ :effective_type :type/DateTime
+ :coercion_strategy :Coercion/ISO8601->DateTime}]
+ (is (=? {:fields [{:name "created_at"
+ :base_type "type/Text"
+ :effective_type "type/DateTime"
+ :coercion_strategy "Coercion/ISO8601->DateTime"}]}
+ (mt/user-http-request :rasta :get 200 (str "agent/v1/table/" table-id))))))))
(deftest list-databases-test
(testing "Returns a list of databases"