The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Fix database tools perf (#72633)
Alexander Polyankin1526a1b26ebf
diff --git a/src/metabase/agent_api/api.clj b/src/metabase/agent_api/api.cljindex 5ea4e9c9750..214869705cd 100644--- a/src/metabase/agent_api/api.clj+++ b/src/metabase/agent_api/api.clj@@ -130,7 +130,7 @@[:database_engine :string][:database_schema {:optional true} [:maybe :string]][:description {:optional true} [:maybe :string]]- [:fields [:sequential ::field]]+ [:fields {:optional true} [:maybe [:sequential ::field]]][:related_tables {:optional true} [:maybe [:sequential ::related-table]]][:metrics {:optional true} [:maybe [:sequential ::metric-summary]]][:measures {:optional true} [:maybe [:sequential ::measure]]]@@ -222,27 +222,16 @@{:scope metabot/agent-database-read:tool {:name "get_database"}}[{:keys [id]} :- [:map [:id ms/PositiveInt]]- {:keys [with-tables with-fields with-field-values with-related-tables with-metrics with-measures with-segments]- :or {with-tables false, with-fields false, with-field-values false, with-related-tables false,- with-metrics false, with-measures false, with-segments false}}+ {:keys [with-tables with-fields]+ :or {with-tables false, with-fields false}}:- [:map- [:with-tables {:optional true} [:maybe :boolean]]- [:with-fields {:optional true} [:maybe :boolean]]- [:with-field-values {:optional true} [:maybe :boolean]]- [:with-related-tables {:optional true} [:maybe :boolean]]- [:with-metrics {:optional true} [:maybe :boolean]]- [:with-measures {:optional true} [:maybe :boolean]]- [:with-segments {:optional true} [:maybe :boolean]]]]+ [:with-tables {:optional true} [:maybe :boolean]]+ [:with-fields {:optional true} [:maybe :boolean]]]](check-tool-result(entity-details/get-database-details- {:database-id id- :with-tables? with-tables- :with-fields? with-fields- :with-field-values? with-field-values- :with-related-tables? with-related-tables- :with-metrics? with-metrics- :with-measures? with-measures- :with-segments? with-segments})))+ {:database-id id+ :with-tables? with-tables+ :with-fields? with-fields})))(api.macros/defendpoint :get "/v1/table/:id" :- ::table"Get details for a table by ID."diff --git a/src/metabase/metabot/tools/entity_details.clj b/src/metabase/metabot/tools/entity_details.cljindex ca574ef5030..8db9b64c6f8 100644--- a/src/metabase/metabot/tools/entity_details.clj+++ b/src/metabase/metabot/tools/entity_details.clj@@ -192,7 +192,7 @@db-id (if metadata-provider (:db-id base) (:db_id base))db-engine (some-> (if metadata-provider(lib.metadata/database metadata-provider)- (metabot.tools.u/get-database db-id :engine))+ (metabot.tools.u/get-database db-id)):engine name)mp (when query-needed?(or metadata-provider@@ -207,18 +207,9 @@(metabot.tools.u/table-field-id-prefix id))related-tables (when with-related-tables?(related-tables table-query field-id-prefix with-fields? field-values-fn))]- (-> {:id id- :type :table- :fields (into [] (map-indexed #(metabot.tools.u/->result-column table-query %2 %1 field-id-prefix)) cols)- :name (:name base)- ;; :display_name should be (lib/display-name table-query), but we want to avoid creating the query if possible- :display_name (some->> (:name base)- (u.humanization/name->human-readable-name :simple))- :database_id db-id- :database_engine db-engine- :database_schema (:schema base)}- (m/assoc-some :description (:description base)- :related_tables related-tables+ (-> (metabot.tools.u/->result-table base db-engine)+ (assoc :fields (into [] (map-indexed #(metabot.tools.u/->result-column table-query %2 %1 field-id-prefix)) cols))+ (m/assoc-some :related_tables related-tables:metrics (when with-metrics?(not-empty (mapv #(convert-metric % mp options)(lib/available-metrics table-query))))@@ -394,46 +385,47 @@{:output (ex-message e) :status-code 404}(metabot.tools.u/handle-agent-error e)))))-(defn- database-details- "Convert a Database model to the agent API response shape."- [db]- {:id (:id db)- :name (:name db)- :engine (some-> (:engine db) name)})-(defn list-databases"Get a list of all databases the current user can read without including its tables."[](try- (let [dbs (metabot.tools.u/list-databases :name :engine)]- {:structured-output (mapv database-details dbs)})+ (let [dbs (metabot.tools.u/list-databases)]+ {:structured-output (mapv metabot.tools.u/->result-database dbs)})(catch Exception e(metabot.tools.u/handle-agent-error e))))+(defn- database-table-details+ "Build a table result map for `get-database-details`, optionally including its fields."+ [table engine table-id->fields with-fields?]+ (let [id (:id table)+ prefix (metabot.tools.u/table-field-id-prefix id)+ cols (->> (get table-id->fields id)+ (sort-by (juxt (fn [f] (or (:position f) 0)) :id)))]+ (cond-> (metabot.tools.u/->result-table table engine)+ with-fields?+ (assoc :fields+ (into []+ (map-indexed+ (fn [i field]+ (metabot.tools.u/->result-column field i prefix)))+ cols)))))+(defn get-database-details- "Get details for a database by ID, optionally including its tables."- [{:keys [database-id with-tables? with-fields? with-field-values? with-related-tables?- with-metrics? with-measures? with-segments?]- :or {with-tables? false, with-fields? false, with-field-values? false, with-related-tables? false,- with-metrics? false, with-measures? false, with-segments? false}}]+ "Get details for a database by ID, optionally including its tables and fields.+ Uses raw t2 queries so it stays efficient on databases with many tables and fields."+ [{:keys [database-id with-tables? with-fields?]+ :or {with-tables? false, with-fields? false}}](try- (lib-be/with-metadata-provider-cache- (let [db (metabot.tools.u/get-database database-id :name :engine)- response (database-details db)]- {:structured-output- (cond-> response- with-tables?- (assoc :tables- (let [mp (lib-be/application-database-metadata-provider database-id)- tables (lib.metadata/tables mp)- opts (cond-> {:metadata-provider mp- :with-fields? with-fields?- :with-related-tables? with-related-tables?- :with-metrics? with-metrics?- :with-measures? with-measures?- :with-segments? with-segments?}- (not with-field-values?) (assoc :field-values-fn identity))]- (mapv #(table-details (:id %) opts) tables))))}))+ (let [db (metabot.tools.u/get-database database-id)+ engine (some-> (:engine db) name)+ tables (when with-tables?+ (metabot.tools.u/list-database-tables database-id))+ table-id->fields (when with-fields?+ (group-by :table_id (metabot.tools.u/list-database-table-fields database-id)))]+ {:structured-output+ (cond-> (metabot.tools.u/->result-database db)+ with-tables?+ (assoc :tables (mapv #(database-table-details % engine table-id->fields with-fields?) tables)))})(catch Exception e(if (= (:status-code (ex-data e)) 404){:output (ex-message e) :status-code 404}diff --git a/src/metabase/metabot/tools/util.clj b/src/metabase/metabot/tools/util.cljindex cf03039067e..2ef3ef39c46 100644--- a/src/metabase/metabot/tools/util.clj+++ b/src/metabase/metabot/tools/util.clj@@ -11,6 +11,7 @@[metabase.models.interface :as mi][metabase.premium-features.core :as premium-features][metabase.util :as u]+ [metabase.util.humanization :as u.humanization][metabase.util.log :as log][toucan2.core :as t2]))@@ -53,27 +54,60 @@"A prefix pattern accepting columns from any entity."#"^.*-(\d+)")+(defn ->result-database+ "Build the agent result map for a database."+ [db]+ {:id (:id db)+ :name (:name db)+ :engine (some-> (:engine db) name)})++(defn ->result-table+ "Build the base agent result map for a table (without `:fields`).+ Accepts either a raw t2 row (underscore keys) or a lib-shaped table (hyphenated keys)."+ [table db-engine]+ (let [table (u/kebab->snake-keys table)]+ (cond-> {:id (:id table)+ :type :table+ :name (:name table)+ :display_name (or (:display_name table)+ (some->> (:name table) (u.humanization/name->human-readable-name :simple)))+ :database_id (:db_id table)+ :database_engine db-engine+ :database_schema (:schema table)}+ (:description table) (assoc :description (:description table)))))+(defn ->result-column- "Return tool result columns for `column` of `query`. The position of `column` is determined by `index`.- The ID is then generated by prefixing the position with `field-id-prefix`."- [query column index field-id-prefix]- (let [base-type (some-> (:base-type column) u/qualified-name)- effective-type (some-> (:effective-type column) u/qualified-name)- semantic-type (some-> (:semantic-type column) u/qualified-name)- coercion-strategy (some-> (:coercion-strategy column) u/qualified-name)]- (-> {:field_id (str field-id-prefix index)- ;; Prefer desired-column-alias over source-column-alias to ensure we use deduplicated names if needed- :name (or (:lib/desired-column-alias column)- (:lib/source-column-alias column))- :display_name (lib/display-name query column)}- (m/assoc-some :description (:description column)- :base_type base-type- :effective_type (when (not= effective-type base-type) effective-type)- :semantic_type semantic-type- :database_type (:database-type column)- :coercion_strategy coercion-strategy- :field_values (:field-values column)- :table_reference (:table-reference column)))))+ "Return a tool result column for `column`. Position is determined by `index`; the field ID is+ `field-id-prefix` + `index`.++ Accepts either a raw t2 row (underscore keys) or a lib-shaped column (hyphenated keys).++ The 3-arity takes the display name directly from the column; the 4-arity computes it via+ `(lib/display-name query column)`."+ ([column index field-id-prefix]+ (let [column (u/kebab->snake-keys column)+ base-type (some-> (:base_type column) u/qualified-name)+ effective-type (some-> (:effective_type column) u/qualified-name)+ semantic-type (some-> (:semantic_type column) u/qualified-name)+ coercion-strategy (some-> (:coercion_strategy column) u/qualified-name)]+ (-> {:field_id (str field-id-prefix index)+ ;; Prefer desired-column-alias over source-column-alias to ensure we use deduplicated names if needed+ :name (or (:lib/desired_column_alias column)+ (:lib/source_column_alias column)+ (:name column))+ :display_name (:display_name column)}+ (m/assoc-some :description (:description column)+ :base_type base-type+ :effective_type (when (not= effective-type base-type) effective-type)+ :semantic_type semantic-type+ :database_type (:database_type column)+ :coercion_strategy coercion-strategy+ :field_values (:field_values column)+ :table_reference (:table_reference column)))))+ ([query column index field-id-prefix]+ (->result-column (assoc column :display-name (lib/display-name query column))+ index+ field-id-prefix)))(defn parse-field-id"Parse a field-id string into its components.@@ -161,18 +195,58 @@second)):schedule_frame (some->> day-of-month name (re-find #"^(?:first|mid|last)"))})+(defn- database-filter+ "HoneySQL WHERE clause matching tables (alias `:t`) belonging to the database with `database-id`."+ [database-id]+ [:= :t.db_id database-id])++(defn- table-filter+ "HoneySQL WHERE clause keeping only user-visible tables (alias `:t`): active and not hidden,+ technical, or cruft."+ []+ [:and [:= :t.active true]+ [:or+ [:= :t.visibility_type nil]+ [:not-in :t.visibility_type [:inline ["hidden" "technical" "cruft"]]]]])++(defn- field-filter+ "HoneySQL WHERE clause keeping only user-visible fields (alias `:f`): active and not retired+ or sensitive."+ []+ [:and [:= :f.active true]+ [:or+ [:= :f.visibility_type nil]+ [:not-in :f.visibility_type [:inline ["retired" "sensitive"]]]]])++(defn list-databases+ "Get all databases the current user can read."+ []+ (->> (t2/select [:model/Database :id :name :engine])+ (filterv mi/can-read?)))+(defn get-database- "Get the `fields` of the database with ID `id`."- [id & fields]- (-> (t2/select-one (into [:model/Database :id] fields) id)+ "Get the database with ID `database-id`."+ [database-id]+ (-> (t2/select-one [:model/Database :id :name :engine] :id database-id)api/read-check))-(defn list-databases- "Get all databases the current user can read, ordered by name."- [& fields]- (->> (t2/select (into [:model/Database :id] fields) {:order-by [[:name :asc]]})+(defn list-database-tables+ "Get all active tables in the database with ID `database-id`."+ [database-id]+ (->> (t2/select [:model/Table :t.id :t.db_id :t.name :t.schema :t.display_name :t.description]+ {:from [[(t2/table-name :model/Table) :t]]+ :where [:and (database-filter database-id) (table-filter)]})(filterv mi/can-read?)))+(defn list-database-table-fields+ "Get all active fields for active tables in the database with ID `database-id`."+ [database-id]+ (t2/select [:model/Field :f.id :f.table_id :f.position :f.name :f.display_name :f.description+ :f.base_type :f.effective_type :f.semantic_type :f.coercion_strategy :f.database_type]+ {:from [[(t2/table-name :model/Field) :f]]+ :join [[(t2/table-name :model/Table) :t] [:= :t.id :f.table_id]]+ :where [:and (database-filter database-id) (table-filter) (field-filter)]}))+(defn get-table"Get the `fields` of the table with ID `id`."[id & fields]diff --git a/test/metabase/agent_api/api_test.clj b/test/metabase/agent_api/api_test.cljindex d3886196c5e..81ef3e1895d 100644--- a/test/metabase/agent_api/api_test.clj+++ b/test/metabase/agent_api/api_test.clj@@ -122,33 +122,42 @@(deftest get-table-details-field-types-test(testing "Field metadata (base_type, effective_type, semantic_type, coercion_strategy) is returned correctly"- (mt/with-temp [:model/Database {db-id :id} {}- :model/Table {table-id :id} {:db_id db-id, :name "t", :active true}- :model/Field _ {:table_id table-id, :name "id"- :base_type :type/BigInteger- :semantic_type :type/PK}- :model/Field _ {:table_id table-id, :name "name"- :base_type :type/Text}- :model/Field _ {:table_id table-id, :name "created_at"- :base_type :type/Text- :effective_type :type/DateTime- :coercion_strategy :Coercion/ISO8601->DateTime}]- (let [fields (-> (mt/user-http-request :rasta :get 200 (str "agent/v1/table/" table-id))- :fields)- by-name (m/index-by :name fields)]- (testing "base_type is always set"- (is (= "type/BigInteger" (get-in by-name ["id" :base_type])))- (is (= "type/Text" (get-in by-name ["name" :base_type])))- (is (= "type/Text" (get-in by-name ["created_at" :base_type]))))- (testing "semantic_type is returned when set"- (is (= "type/PK" (get-in by-name ["id" :semantic_type])))- (is (nil? (get-in by-name ["name" :semantic_type]))))- (testing "effective_type and coercion_strategy are returned when coerced"- (is (= "type/DateTime" (get-in by-name ["created_at" :effective_type])))- (is (= "Coercion/ISO8601->DateTime" (get-in by-name ["created_at" :coercion_strategy]))))- (testing "effective_type is omitted when it equals base_type"- (is (not (contains? (get by-name "id") :effective_type)))- (is (not (contains? (get by-name "name") :effective_type))))))))+ (testing "base_type is always set, semantic_type is returned when set"+ (mt/with-temp [:model/Database {db-id :id} {}+ :model/Table {table-id :id} {:db_id db-id, :name "t", :active true}+ :model/Field _ {:table_id table-id, :name "id"+ :base_type :type/BigInteger+ :semantic_type :type/PK}]+ (is (=? {:fields [{:name "id"+ :base_type "type/BigInteger"+ :semantic_type "type/PK"}]}+ (mt/user-http-request :rasta :get 200 (str "agent/v1/table/" table-id))))))++ (testing "semantic_type is omitted when not set, effective_type is omitted when it equals base_type"+ (mt/with-temp [:model/Database {db-id :id} {}+ :model/Table {table-id :id} {:db_id db-id, :name "t", :active true}+ :model/Field _ {:table_id table-id, :name "name"+ :base_type :type/Text}]+ (let [[field :as fields] (:fields (mt/user-http-request :rasta :get 200 (str "agent/v1/table/" table-id)))]+ (is (= 1 (count fields)))+ (is (=? {:name "name"+ :base_type "type/Text"}+ field))+ (is (not (contains? field :semantic_type)))+ (is (not (contains? field :effective_type))))))++ (testing "effective_type and coercion_strategy are returned when coerced"+ (mt/with-temp [:model/Database {db-id :id} {}+ :model/Table {table-id :id} {:db_id db-id, :name "t", :active true}+ :model/Field _ {:table_id table-id, :name "created_at"+ :base_type :type/Text+ :effective_type :type/DateTime+ :coercion_strategy :Coercion/ISO8601->DateTime}]+ (is (=? {:fields [{:name "created_at"+ :base_type "type/Text"+ :effective_type "type/DateTime"+ :coercion_strategy "Coercion/ISO8601->DateTime"}]}+ (mt/user-http-request :rasta :get 200 (str "agent/v1/table/" table-id))))))))(deftest list-databases-test(testing "Returns a list of databases"