The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Address the remaining db.clj review findings: metabot, sync, search and friends (#81865)
Alexander Polyankina865fd1ad737
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 34fae22741b..88b24b589af 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -407,8 +407,7 @@auth-identity{:team "UX West"- :api #{metabase.auth-identity.core- metabase.auth-identity.init}+ :api #{metabase.auth-identity.core metabase.auth-identity.db metabase.auth-identity.init}:uses #{channeleventslogin-history@@ -1499,7 +1498,8 @@models{:team "DevEx"- :api #{metabase.models.humanization+ :api #{metabase.models.db+ metabase.models.humanizationmetabase.models.initmetabase.models.interfacemetabase.models.resolutiondiff --git a/enterprise/backend/src/metabase_enterprise/stale/api.clj b/enterprise/backend/src/metabase_enterprise/stale/api.cljindex 5b751bbda3f..6a9a2d35445 100644--- a/enterprise/backend/src/metabase_enterprise/stale/api.clj+++ b/enterprise/backend/src/metabase_enterprise/stale/api.clj@@ -25,7 +25,7 @@(comp (map :id)(remove #(= % (collection/trash-collection-id)))(filter #(collection/visible-collection-id? % :write)))- (collection/descendants-flat collection [:= :archived false])))+ (collection/descendants-flat collection false)))(defmulti present-model-items"Given a model and a list of items, return the items in the format the API client expects. Note that order does notdiff --git a/src/metabase/auth_identity/db.clj b/src/metabase/auth_identity/db.cljindex e08a3d53ae8..f5dd76b0770 100644--- a/src/metabase/auth_identity/db.clj+++ b/src/metabase/auth_identity/db.clj@@ -46,7 +46,9 @@(t2/delete! :model/AuthIdentity :user_id user-id :provider provider))(defn delete-sessions-for-user!- "Delete every Session of the User with `user-id`."+ "Delete every Session of the User with `user-id`. Duplicates `metabase.session.db/delete-sessions-for-user!`;+ can't delegate to it because the `session` module already depends on `auth-identity`, so the reverse dependency+ would be a module cycle."[user-id](t2/delete! :model/Session :user_id user-id))diff --git a/src/metabase/collections/db.clj b/src/metabase/collections/db.cljindex 1aecfce4fe0..031301eef72 100644--- a/src/metabase/collections/db.clj+++ b/src/metabase/collections/db.clj@@ -105,10 +105,24 @@[collection-ids](t2/select [:model/Collection :name :id :personal_owner_id] :id [:in collection-ids] {:order-by [:location]}))-(defn descendant-summaries-where- "The name, ID, location, and description of the Collections matching the Honey SQL `where`."- [where]- (t2/select [:model/Collection :name :id :location :description] {:where where}))+(defn descendant-summaries+ "The name, ID, location, and description of the descendant Collections of the Collection at+ `children-location-prefix` (see `metabase.collections.models.collection/children-location`), excluding other+ users' Personal Collections (Personal Collections owned by `current-user-id` are still included). When+ `archived?` is given (true or false, not nil), further restricted to that archived status.+ `additional-where-clauses` are ANDed in as-is; used by callers that need a permission-filter builder like+ `visible-collection-filter-clause`, which lives in `metabase.collections.models.collection` and so can't be+ called from here without a require cycle (that namespace already requires this one)."+ [children-location-prefix current-user-id archived? additional-where-clauses]+ (t2/select [:model/Collection :name :id :location :description]+ {:where (into [:and+ [:like :location (str children-location-prefix "%")]+ [:or+ [:= :personal_owner_id nil]+ [:= :personal_owner_id current-user-id]]+ (when (some? archived?)+ [:= :archived archived?])]+ additional-where-clauses)}))(defn descendant-summaries-with-type"The name, ID, location, description, and type of the Collections directly under any of `location-prefixes`diff --git a/src/metabase/collections/models/collection.clj b/src/metabase/collections/models/collection.cljindex eaee77b83a1..587827335ee 100644--- a/src/metabase/collections/models/collection.clj+++ b/src/metabase/collections/models/collection.clj@@ -1117,24 +1117,17 @@[:ref ::children]])(mu/defn descendants-flat :- [:sequential CollectionWithLocationAndIDOrRoot]- "Return all descendant collections of a `collection`, including children, grandchildren, and so forth."- [collection :- CollectionWithLocationAndIDOrRoot, & additional-honeysql-where-clauses]- (or- (collections.db/descendant-summaries-where- (apply- vector- :and- [:like :location (str (children-location collection) "%")]- ;; Only return the Personal Collection belonging to the Current- ;; User, regardless of whether we should actually be allowed to see- ;; it (e.g., admins have perms for all Collections). This is done- ;; to keep the Root Collection View for admins from getting crazily- ;; cluttered with Personal Collections belonging to other users- [:or- [:= :personal_owner_id nil]- [:= :personal_owner_id api/*current-user-id*]]- additional-honeysql-where-clauses))- []))+ "Return all descendant collections of a `collection`, including children, grandchildren, and so forth. Excludes+ other users' Personal Collections, regardless of whether we should actually be allowed to see them (e.g. admins+ have perms for all Collections) -- this is done to keep the Root Collection View for admins from getting crazily+ cluttered with Personal Collections belonging to other users. `archived?`, if given (true or false), restricts+ to Collections with that archived status. `additional-honeysql-where-clauses` are for permission-filter builders+ like [[visible-collection-filter-clause]]."+ ([collection :- CollectionWithLocationAndIDOrRoot]+ (descendants-flat collection nil))+ ([collection :- CollectionWithLocationAndIDOrRoot, archived? :- [:maybe :boolean], & additional-honeysql-where-clauses]+ (collections.db/descendant-summaries+ (children-location collection) api/*current-user-id* archived? additional-honeysql-where-clauses)))(mu/defn descendants-flat-for :- [:sequential CollectionWithLocationAndIDOrRoot]"Like [[descendants-flat]], but returns the descendants of *any* of@@ -1160,10 +1153,10 @@where each letter represents a Collection, and the arrows represent values of its respective `:children`set."- [collection :- CollectionWithLocationAndIDOrRoot, & additional-honeysql-where-clauses]+ [collection :- CollectionWithLocationAndIDOrRoot];; first, fetch all the descendants of the `collection`, and build a map of location -> children. This will be used;; so we can fetch the immediate children of each Collection- (let [location->children (group-by :location (apply descendants-flat collection additional-honeysql-where-clauses))+ (let [location->children (group-by :location (descendants-flat collection));; Next, build a function to add children to a given `coll`. This function will recursively call itself to add;; children to each childadd-children (fn add-children [coll]diff --git a/src/metabase/collections_rest/api.clj b/src/metabase/collections_rest/api.cljindex 6851258e24e..776e57298de 100644--- a/src/metabase/collections_rest/api.clj+++ b/src/metabase/collections_rest/api.clj@@ -839,9 +839,10 @@(defn- annotate-collections[parent-coll colls {:keys [show-dashboard-questions?]}]- (let [descendant-collections (collection/descendants-flat parent-coll (collection/visible-collection-filter-clause- :id- {:include-archived-items :all}))+ (let [descendant-collections (collection/descendants-flat parent-coll nil+ (collection/visible-collection-filter-clause+ :id+ {:include-archived-items :all}))descendant-collection-ids (mapv u/the-id descendant-collections)diff --git a/src/metabase/metabot/api/metabot.clj b/src/metabase/metabot/api/metabot.cljindex 173815c7b01..bc6ab44aeff 100644--- a/src/metabase/metabot/api/metabot.clj+++ b/src/metabase/metabot/api/metabot.clj@@ -4,11 +4,9 @@[metabase.api.common :as api][metabase.api.macros :as api.macros][metabase.api.routes.common :refer [+auth]]- [metabase.app-db.core :as mdb][metabase.metabot.db :as metabot.db][metabase.metabot.suggested-prompts :as metabot.suggested-prompts][metabase.metabot.task.suggested-prompts-refresh :as metabot.suggested-prompts-refresh]- [metabase.metabot.tools.util :as metabot.tools.u][metabase.metabot.usage :as metabot.usage][metabase.premium-features.core :as premium-features][metabase.request.core :as request]@@ -100,30 +98,9 @@[:sample {:optional true} :boolean][:model {:optional true} [:enum "metric" "model"]][:model_id {:optional true} pos-int?]]]- (let [offset (when-not sample (request/offset))- rand-fn (case (mdb/db-type)- :postgres :random- :rand)- base-query (cond-> {:join [[^:allow-subquery {:select [:id :name :type]- :from [[(metabot.tools.u/metabot-metrics-and-models-query id)- :scope]]}- :card]- [:and- [:= :card.id :metabot_prompt.card_id]]]- :where [:and- [:= :metabot_prompt.metabot_id id]]}- model (update :where conj [:= :card.type model])- model_id (update :where conj [:= :card.id model_id]))- total (metabot.db/prompt-count-where base-query)- order-by (if sample- [[[rand-fn]]]- [[:card.name :asc]- [:id :asc]])- prompts (metabot.db/prompts-where- (cond-> base-query- true (assoc :order-by order-by)- (request/limit) (assoc :limit (request/limit))- offset (assoc :offset offset)))]+ (let [offset (when-not sample (request/offset))+ total (metabot.db/prompt-count id model model_id)+ prompts (metabot.db/prompts id model model_id sample (request/limit) offset)]{:prompts prompts:limit (request/limit):offset offsetdiff --git a/src/metabase/metabot/db.clj b/src/metabase/metabot/db.cljindex 2b441d126d8..1c1d0d4c040 100644--- a/src/metabase/metabot/db.clj+++ b/src/metabase/metabot/db.clj@@ -3,8 +3,18 @@additional logic, so the rest of the module only touches `toucan2.core` for model definitions, hydration methods,and transactions."(:require+ [clojure.string :as str]+ [metabase.api.common :as api]+ [metabase.app-db.core :as mdb]+ [metabase.audit-app.core :as audit-app]+ [metabase.collections.models.collection :as collection.model]+ [metabase.models.interface :as mi]+ [metabase.premium-features.core :as premium-features]+ [metabase.util :as u][toucan2.core :as t2]))+(declare collection metabot-metrics-and-models-query root-collections-of-types)+;;; ------------------------------------------------- Metabot -------------------------------------------------(defn metabot@@ -44,9 +54,32 @@[metabot-ids](t2/select :model/MetabotPrompt {:where [:in :metabot_id metabot-ids]}))-(defn prompts-where- "The prompt, model, and Card columns of the MetabotPrompts matching the Honey SQL `query`."- [query]+(defn- prompts-for-metabot-query+ "Honey SQL `:join`/`:where` restricting to MetabotPrompts of the Metabot with `metabot-id` whose Card is within+ scope, optionally further restricted to Cards of `card-type` or the Card with `card-id`."+ [metabot-id card-type card-id]+ (cond-> {:join [[^:allow-subquery {:select [:id :name :type]+ :from [[(metabot-metrics-and-models-query metabot-id) :scope]]}+ :card]+ [:and+ [:= :card.id :metabot_prompt.card_id]]]+ :where [:and+ [:= :metabot_prompt.metabot_id metabot-id]]}+ card-type (update :where conj [:= :card.type card-type])+ card-id (update :where conj [:= :card.id card-id])))++(defn- prompt-sample-order-by+ "A random `:order-by` clause, using the database's native random function."+ []+ [[[(case (mdb/db-type)+ :postgres :random+ :rand)]]])++(defn prompts+ "The prompt, model, and Card columns of the MetabotPrompts of the Metabot with `metabot-id` whose Card is within+ scope, optionally restricted to Cards of `card-type` or the Card with `card-id`, ordered randomly if `sample?` else+ by Card name, and limited/offset by `limit`/`offset`."+ [metabot-id card-type card-id sample? limit offset](t2/select [:model/MetabotPrompt:id:prompt@@ -55,12 +88,18 @@[:card.name :model_name]:created_at:updated_at]- query))--(defn prompt-count-where- "The number of MetabotPrompts matching the Honey SQL `query`."- [query]- (t2/count :model/MetabotPrompt query))+ (cond-> (prompts-for-metabot-query metabot-id card-type card-id)+ true (assoc :order-by (if sample?+ (prompt-sample-order-by)+ [[:card.name :asc] [:id :asc]]))+ limit (assoc :limit limit)+ offset (assoc :offset offset))))++(defn prompt-count+ "The number of MetabotPrompts of the Metabot with `metabot-id` whose Card is within scope, optionally restricted to+ Cards of `card-type` or the Card with `card-id`."+ [metabot-id card-type card-id]+ (t2/count :model/MetabotPrompt (prompts-for-metabot-query metabot-id card-type card-id)))(defn prompt-count-for-metabot"The number of MetabotPrompts of the Metabot with `metabot-id`."@@ -358,36 +397,59 @@:active true:visibility_type nil))-(defn visible-table-summaries-where+(defn- current-user-visible-table-clause+ "Honey SQL `{:where …}` (plus `:with` when the filter needs a CTE) restricting Tables to those visible to the+ current user for querying."+ []+ (let [{table-where-clause :clause table-cte :with}+ (mi/visible-filter-clause :model/Table+ :id+ {:user-id api/*current-user-id*+ :is-superuser? api/*is-superuser?*}+ {:perms/view-data :unrestricted+ :perms/create-queries :query-builder-and-native})]+ (cond-> {:where table-where-clause}+ table-cte (assoc :with table-cte))))++(defn visible-table-summaries-for-current-user"The ID, name, schema, and description of the active, unhidden Tables among `table-ids` in the Database with- `database-id` matching the Honey SQL `query`."- [database-id table-ids query]+ `database-id` that are visible to the current user for querying."+ [database-id table-ids](t2/select [:model/Table :id :name :schema :description]:db_id database-id:id [:in table-ids]:active true:visibility_type nil- query))+ (current-user-visible-table-clause)))++(def ^:private max-visible-tables-to-consider+ "Cap on the number of visible Tables fetched for fuzzy table-name matching."+ 10000)-(defn visible-tables-reducible- "Reducible ID, name, schema, and description of the active, unhidden Tables in the Database with `database-id`- matching the Honey SQL `query`."- [database-id query]+(defn visible-tables-excluding+ "Reducible ID, name, schema, and description of up to [[max-visible-tables-to-consider]] active, unhidden Tables in+ the Database with `database-id` that are visible to the current user for querying, excluding `excluded-table-ids`."+ [database-id excluded-table-ids](t2/reducible-select [:model/Table :id :name :schema :description]:db_id database-id:active true:visibility_type nil- query))--(defn most-viewed-tables-where- "The ID, Database ID, name, schema, and description of the active, unhidden Tables in the Database with- `database-id` matching the Honey SQL `query`."- [database-id query]+ (cond-> (assoc (current-user-visible-table-clause) :limit max-visible-tables-to-consider)+ (seq excluded-table-ids)+ (update :where (fn [where-clause]+ (if where-clause+ [:and where-clause [:not-in :id excluded-table-ids]]+ [:not-in :id excluded-table-ids]))))))++(defn most-viewed-tables-visible-to-current-user+ "The ID, Database ID, name, schema, and description of up to `limit` active, unhidden Tables in the Database with+ `database-id` that are visible to the current user for querying, most viewed first."+ [database-id limit](t2/select [:model/Table :id :db_id :name :schema :description]:db_id database-id:active true:visibility_type nil- query))+ (assoc (current-user-visible-table-clause) :order-by [[:view_count :desc]] :limit limit)))(defn table-names"Up to `limit` IDs, names, and schemas of the active, unhidden Tables in the Database with `database-id`."@@ -423,21 +485,60 @@:where [:and [:= :db_id database-id] [:= :active true]]:order-by [[:schema :asc]]}))-(defn query-table-reference-where- "The first Table ID, name, and schema matching the Honey SQL `where`, as a query table reference."- [where]+(def ^:private quoted-identifier-chars+ "Characters that quote a SQL identifier, making its match against a Table name/schema case-sensitive."+ "\"`")++(defn- quote-stripper+ [quote-char]+ (let [doubled (str quote-char quote-char)+ single (str quote-char)]+ #(-> (subs % 1 (dec (count %)))+ (str/replace doubled single))))++(def ^:private quote-char->stripper+ (zipmap quoted-identifier-chars (map quote-stripper quoted-identifier-chars)))++(defn- table-part-clause+ "Exact match for a quoted `value`, case-insensitive match for an unquoted `value`. Case-insensitive matching is not+ correct for every database (Oracle/Postgres/H2 all treat unquoted identifiers differently), but MySQL is truly+ case-insensitive, so this caters to the lowest common denominator; identifiers differing only by case are already+ an anti-pattern, so this leniency is unlikely to cause issues in practice."+ [field value]+ (if-let [strip (quote-char->stripper (first value))]+ [:= field (strip value)]+ [:= [:lower field] (u/lower-case-en value)]))++(defn- table-match-clause+ [{:keys [schema table]}]+ (if-not schema+ (table-part-clause :t.name table)+ [:and+ (table-part-clause :t.name table)+ (table-part-clause :t.schema schema)]))++(defn query-table-reference+ "The first Table ID, name, and schema in the Database with `db-id` matching `table` (and `schema`, if given), as a+ query table reference. Matching is case-insensitive unless `table`/`schema` are quoted with `\"` or `` ` ``."+ [db-id schema table](t2/select-one :model/QueryTable{:select [[:t.id :table-id] [:t.name :table] [:t.schema :schema]]:from [[(t2/table-name :model/Table) :t]]- :where where}))--(defn query-table-references-where- "The Table IDs, names, and schemas matching the Honey SQL `where`, as query table references."- [where]+ :where [:and+ [:= :t.db_id db-id]+ (table-match-clause {:schema schema :table table})]}))++(defn query-table-references+ "The Table IDs, names, and schemas in the Database with `db-id` matching any of `tables` (each a map of `:schema`+ and `:table`), as query table references. Matching is case-insensitive unless quoted, as in+ [[query-table-reference]]."+ [db-id tables](t2/select :model/QueryTable{:select [[:t.id :table-id] [:t.name :table] [:t.schema :schema]]:from [[(t2/table-name :model/Table) :t]]- :where where}))+ :where [:and+ [:= :t.db_id db-id]+ (into [:or] (map table-match-clause) tables)]}));;; ------------------------------------------------- Fields -------------------------------------------------@@ -478,6 +579,71 @@[card-id](t2/select-one [:model/Card :id :type :card_schema] :id card-id))+(defn metabot-metrics-and-models-query+ "Honey SQL query selecting the metric and model Cards in scope of the Metabot with `metabot-id` that are visible to+ the current user, ignoring analytics content. If the Metabot has `:use_verified_content` enabled, restricts to+ verified-or-curated content (verified, official-collection, or library-published). `limit`, if given, caps the+ number of rows."+ [metabot-id & {:keys [limit]}]+ (let [metabot-instance (metabot metabot-id)+ metabot-collection-id (:collection_id metabot-instance)+ use-verified-content? (:use_verified_content metabot-instance)+ verified? (premium-features/has-feature? :content-verification)+ official? (premium-features/has-feature? :official-collections)+ library? (premium-features/has-feature? :library)+ ;; ids of collections under a Library-type root; their metrics/models are library-published content+ library-coll-ids (when library?+ (let [roots (root-collections-of-types (mapv name collection.model/library-collection-types))]+ (into (set (map :id roots)) (mapcat collection.model/descendant-ids roots))))+ ;; Mirror collections.curation/curated? for card scope: verified, official-collection, or+ ;; library-published (under a Library root). Each disjunct is gated on its feature.+ curated-conds (cond-> []+ verified? (conj [:= :mr.status "verified"])+ official? (conj [:= :collection.authority_level "official"])+ (seq library-coll-ids) (conj [:in :report_card.collection_id (vec library-coll-ids)]))+ ;; Columns are qualified with report_card because the official-collections branch joins+ ;; `collection`, which shares column names (type, archived, id) — unqualified refs would be ambiguous.+ collection-filter (if metabot-collection-id+ (let [metabot-collection (collection metabot-collection-id)+ collection-ids (conj (collection.model/descendant-ids metabot-collection) metabot-collection-id)]+ [:in :report_card.collection_id collection-ids])+ [:and true])+ base-query ^:allow-subquery {:select [:report_card.*]+ :from [[:report_card]]+ :where [:and+ [:!= :report_card.database_id audit-app/audit-db-id]+ collection-filter+ [:in :report_card.type ["metric" "model"]]+ [:= :report_card.archived false]+ (when api/*current-user-id*+ (collection.model/visible-collection-filter-clause :report_card.collection_id))]}]+ (cond-> base-query+ verified?+ (update :left-join (fnil into []) [[:moderation_review :mr] [:and+ [:= :mr.moderated_item_id :report_card.id]+ [:= :mr.moderated_item_type "card"]+ [:= :mr.most_recent true]]])++ official?+ (update :left-join (fnil into []) [[:collection :collection]+ [:= :collection.id :report_card.collection_id]])++ ;; Prioritize curated content.+ (seq curated-conds)+ (assoc :order-by [[[:case (into [:or] curated-conds) [:inline 0] :else [:inline 1]] :asc]])++ ;; Restrict to curated content only when that's desired.+ (and use-verified-content? (seq curated-conds))+ (update :where conj (into [:or] curated-conds))++ ;; Setting on but no curation features active → nothing is curated, so return nothing rather than+ ;; falling through unfiltered to uncurated cards.+ (and use-verified-content? (empty? curated-conds))+ (update :where conj [:= [:inline 1] [:inline 0]])++ (integer? limit)+ (assoc :limit limit))))+(defn cards-where"The Cards matching the Honey SQL `query`."[query]diff --git a/src/metabase/metabot/query_analyzer.clj b/src/metabase/metabot/query_analyzer.cljindex 8cca3914868..3146a76ff22 100644--- a/src/metabase/metabot/query_analyzer.clj+++ b/src/metabase/metabot/query_analyzer.clj@@ -36,41 +36,12 @@(defn- normalized-key [value](u/lower-case-en (strip-quotes value)))-(defn- field-query- "Exact match for quoted fields, case-insensitive match for non-quoted fields"- [field value]- (if-let [f (quote->stripper (first value))]- [:= field (f value)]- ;; Technically speaking, this is not correct for all databases.- ;;- ;; For example, Oracle treats non-quoted identifiers as uppercase, but still expects a case-sensitive match.- ;; Similarly, Postgres treats all non-quoted identifiers as lowercase, and again expects an exact match.- ;; H2 on the other hand will choose whether to cast it to uppercase or lowercase based on a system variable... T_T- ;;- ;; MySQL, by contrast, is truly case-insensitive, and as the lowest common denominator it's what we cater for.- ;; In general, it's a huge anti-pattern to have any identifiers that differ only by case, so this extra leniency is- ;; unlikely to ever cause issues in practice.- ;;- ;; If we want 100% correctness, we can use the Macaw :case-insensitive option here to do the right thing.- [:= [:lower field] (u/lower-case-en value)]))--(defn- table-query- [{:keys [schema table]}]- (if-not schema- (field-query :t.name table)- [:and- (field-query :t.name table)- (field-query :t.schema schema)]))-(defn table-reference"Used by tests"([db-id table](table-reference db-id nil table))([db-id schema table]- (metabot.db/query-table-reference-where [:and- [:= :t.db_id db-id]- (table-query {:schema (some-> schema name)- :table (name table)})])))+ (metabot.db/query-table-reference db-id (some-> schema name) (name table))))(defn- strip-redundant-table-refs"Strip out duplicate references, and unqualified references that are shadowed by found or qualified ones."@@ -100,9 +71,7 @@(consolidate-tablestables(when (seq tables)- (metabot.db/query-table-references-where [:and- [:= :t.db_id db-id]- (into [:or] (map table-query tables))]))))+ (metabot.db/query-table-references db-id tables))))(defn- tables-via-sql-tools"Returns a set of table identifiers that (may) be referenced in the given card's query.diff --git a/src/metabase/metabot/table_utils.clj b/src/metabase/metabot/table_utils.cljindex 2ec46c415a9..a47a52fc49a 100644--- a/src/metabase/metabot/table_utils.clj+++ b/src/metabase/metabot/table_utils.clj@@ -2,10 +2,8 @@"Shared table utilities for enterprise modules."(:require[clojure.set :as set]- [metabase.api.common :as api][metabase.metabot.db :as metabot.db][metabase.metabot.query-analyzer :as query-analyzer]- [metabase.models.interface :as mi][metabase.util :as u][toucan2.core :as t2][toucan2.realize :as t2.realize])@@ -40,18 +38,8 @@priority-tables []exclude-table-ids #{}}}](let [priority-table-ids (set (map :id priority-tables))- {table-where-clause :clause table-cte :with} (mi/visible-filter-clause :model/Table- :id- {:user-id api/*current-user-id*- :is-superuser? api/*is-superuser?*}- {:perms/view-data :unrestricted- :perms/create-queries :query-builder-and-native});; Fetch most viewed tables, excluding priority tables and excluded tables- fill-tables (metabot.db/most-viewed-tables-where database-id- (cond-> {:where table-where-clause- :order-by [[:view_count :desc]]- :limit all-tables-limit}- table-cte (assoc :with table-cte)))+ fill-tables (metabot.db/most-viewed-tables-visible-to-current-user database-id all-tables-limit)fill-tables (remove #(or (priority-table-ids (:id %))(exclude-table-ids (:id %))) fill-tables)fill-tables (t2/hydrate fill-tables :fields)@@ -108,17 +96,6 @@(nil? tschema)(similar? uschema tschema))))-(defn- visible-filter-clause- []- (let [{table-where-clause :clause table-cte :with} (mi/visible-filter-clause :model/Table- :id- {:user-id api/*current-user-id*- :is-superuser? api/*is-superuser?*}- {:perms/view-data :unrestricted- :perms/create-queries :query-builder-and-native})]- (cond-> {:where table-where-clause}- table-cte (assoc :with table-cte))))-(defn find-matching-tables"Find tables in the database that are similar to the unrecognized tables using fuzzy matching.@@ -139,12 +116,7 @@(keep (fn [table](when (some #(matching-tables? table % {:match-schema? false}) unrecognized-tables)(t2.realize/realize table))))- (metabot.db/visible-tables-reducible database-id- (cond-> (assoc (visible-filter-clause)- :limit 10000)- (seq used-ids) (update :where #(if %- [:and % [:not-in :id used-ids]]- [:not-in :id used-ids]))))))+ (metabot.db/visible-tables-excluding database-id used-ids)))(defn used-tables-from-ids"Return table info for `table-ids` in the same shape as [[used-tables]].@@ -153,7 +125,7 @@[database-id table-ids](if-not (seq table-ids)[]- (metabot.db/visible-table-summaries-where database-id table-ids (visible-filter-clause))))+ (metabot.db/visible-table-summaries-for-current-user database-id table-ids)))(defn used-tables"Return all tables used in the query, including fuzzy-matched ones.diff --git a/src/metabase/metabot/tools/util.clj b/src/metabase/metabot/tools/util.cljindex 99419fbfc68..12b60bb8f00 100644--- a/src/metabase/metabot/tools/util.clj+++ b/src/metabase/metabot/tools/util.clj@@ -2,14 +2,11 @@(:require[medley.core :as m][metabase.api.common :as api]- [metabase.audit-app.core :as audit-app]- [metabase.collections.models.collection :as collection][metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.lib.types.isa :as lib.types.isa][metabase.metabot.db :as metabot.db]- [metabase.premium-features.core :as premium-features][metabase.util :as u]))(defn handle-agent-error@@ -193,74 +190,6 @@(let [mp (lib-be/application-database-metadata-provider (:db_id table))](lib/query mp (lib.metadata/table mp table-id)))))-(defn metabot-metrics-and-models-query- "Return the metric and model cards in metabot scope visible to the current user.-- Takes a metabot-id and returns all metric and model cards in that metabot's collection- and its subcollections. If the metabot has use_verified_content enabled, only verified-or-curated- content is returned — verified, official-collection, or library-published cards.-- Ignores analytics content."- [metabot-id & {:keys [limit] :as _opts}]- (let [metabot (metabot.db/metabot metabot-id)- metabot-collection-id (:collection_id metabot)- use-verified-content? (:use_verified_content metabot)- verified? (premium-features/has-feature? :content-verification)- official? (premium-features/has-feature? :official-collections)- library? (premium-features/has-feature? :library)- ;; ids of collections under a Library-type root; their metrics/models are library-published content- library-coll-ids (when library?- (let [roots (metabot.db/root-collections-of-types (mapv name collection/library-collection-types))]- (into (set (map :id roots)) (mapcat collection/descendant-ids roots))))- ;; Mirror collections.curation/curated? for card scope: verified, official-collection, or- ;; library-published (under a Library root). Each disjunct is gated on its feature.- curated-conds (cond-> []- verified? (conj [:= :mr.status "verified"])- official? (conj [:= :collection.authority_level "official"])- (seq library-coll-ids) (conj [:in :report_card.collection_id (vec library-coll-ids)]))- ;; Columns are qualified with report_card because the official-collections branch joins- ;; `collection`, which shares column names (type, archived, id) — unqualified refs would be ambiguous.- collection-filter (if metabot-collection-id- (let [collection (metabot.db/collection metabot-collection-id)- collection-ids (conj (collection/descendant-ids collection) metabot-collection-id)]- [:in :report_card.collection_id collection-ids])- [:and true])- base-query ^:allow-subquery {:select [:report_card.*]- :from [[:report_card]]- :where [:and- [:!= :report_card.database_id audit-app/audit-db-id]- collection-filter- [:in :report_card.type ["metric" "model"]]- [:= :report_card.archived false]- (when api/*current-user-id*- (collection/visible-collection-filter-clause :report_card.collection_id))]}]- (cond-> base-query- verified?- (update :left-join (fnil into []) [[:moderation_review :mr] [:and- [:= :mr.moderated_item_id :report_card.id]- [:= :mr.moderated_item_type "card"]- [:= :mr.most_recent true]]])-- official?- (update :left-join (fnil into []) [[:collection :collection]- [:= :collection.id :report_card.collection_id]])-- ;; Prioritize curated content.- (seq curated-conds)- (assoc :order-by [[[:case (into [:or] curated-conds) [:inline 0] :else [:inline 1]] :asc]])-- ;; Restrict to curated content only when that's desired.- (and use-verified-content? (seq curated-conds))- (update :where conj (into [:or] curated-conds))-- ;; Setting on but no curation features active → nothing is curated, so return nothing rather than- ;; falling through unfiltered to uncurated cards.- (and use-verified-content? (empty? curated-conds))- (update :where conj [:= [:inline 1] [:inline 0]])-- (integer? limit)- (assoc :limit limit))))-(defn- destination-db-ids"Returns the subset of `db-ids` that back a destination (routed) database -- routing internalsreachable only through their router database (see@@ -275,7 +204,7 @@Only cards visible to the current user are returned, excluding those backed by a destination(routed) database (see [[destination-db-ids]])."[metabot-id & {:as opts}]- (let [cards (metabot.db/cards-where (-> (metabot-metrics-and-models-query metabot-id opts)+ (let [cards (metabot.db/cards-where (-> (metabot.db/metabot-metrics-and-models-query metabot-id opts);; qualified: the official-collections branch joins `collection`,;; which also has `id`(update :order-by (fnil conj []) [:report_card.id])))diff --git a/src/metabase/models/db.clj b/src/metabase/models/db.cljindex a7f2ff298ce..7c5f45ef2aa 100644--- a/src/metabase/models/db.clj+++ b/src/metabase/models/db.clj@@ -203,6 +203,13 @@(when (seq field-names)(t2/select-one-pk :model/Field (field-in-path-query table-id field-names))))+(defn field-in-path+ "The Field named by the last of `field-names` (each nested inside the previous, bottom-most first) under+ `table-id`, or nil."+ [table-id field-names]+ (when (seq field-names)+ (t2/select-one :model/Field (field-in-path-query table-id field-names))))+(defn insert-inactive-field!"Insert an inactive, untyped Field named `field-name` under `parent-id` in the Table with `table-id` and return itsid."diff --git a/src/metabase/search/appdb/core.clj b/src/metabase/search/appdb/core.cljindex 7a3301e6ffc..15ed88f1a6c 100644--- a/src/metabase/search/appdb/core.clj+++ b/src/metabase/search/appdb/core.clj@@ -173,7 +173,7 @@scorers (search.scoring/scorers search-ctx)query (->> (base-filtered-query search-ctx search-string [:legacy_input])(search.scoring/with-scores search-ctx scorers))]- (->> (search.db/rows query)+ (->> (search.db/scored-search-rows query)(map (partial rehydrate weights (keys scorers)))))(catch Exception e;; Rule out the error coming from stale index metadata.@@ -197,7 +197,7 @@:search_index.model:search_index.source_type)))(search.filter/with-filters search-ctx)- search.db/rows+ search.db/distinct-model-rows(into #{} (map :model)))))(defn- restrict-to-row [model id qry]@@ -209,7 +209,7 @@(-> qry(assoc :select [[[:inline 1] :one]] :limit 1)(dissoc :order-by)- search.db/rows+ search.db/search-index-probe-rowsseqboolean))diff --git a/src/metabase/search/appdb/index.clj b/src/metabase/search/appdb/index.cljindex b6b08abc551..3dde711bded 100644--- a/src/metabase/search/appdb/index.clj+++ b/src/metabase/search/appdb/index.clj@@ -2,7 +2,6 @@(:require[clojure.set :as set][clojure.string :as str]- [honey.sql.helpers :as sql.helpers][metabase.analytics-interface.core :as analytics][metabase.app-db.core :as mdb][metabase.config.core :as config]@@ -114,7 +113,7 @@(defn- drop-table! [table](boolean(when table- (search.db/execute! (sql.helpers/drop-table :if-exists (keyword (table-name table)))))))+ (search.db/drop-search-index-table-if-exists! (keyword (table-name table))))))(defn- orphan-indexes [](map (comp keyword u/lower-case-en :table_name)@@ -127,7 +126,7 @@(let [dropped (volatile! [])](doseq [table (orphan-indexes)](try- (search.db/execute! (sql.helpers/drop-table table))+ (search.db/drop-search-index-table! table)(vswap! dropped conj table);; Deletion could fail if it races with other instances(catch Exception e@@ -178,12 +177,10 @@;; Create with a separate transaction so that postgresql will complete the index creations before returning,;; even when already running in a transaction(t2/with-transaction [_ (mdb/app-db)]- (-> (sql.helpers/create-table table-name)- (sql.helpers/with-columns (specialization/table-schema base-schema))- search.db/execute!)+ (search.db/create-search-index-table! table-name (specialization/table-schema base-schema))(let [table-name (name table-name)](doseq [stmt (specialization/post-create-statements table-name table-name)]- (search.db/execute! stmt)))))+ (search.db/run-search-index-statement! stmt)))))(defn maybe-create-pending!"Create a search index table if one doesn't exist. Record and return the name of the table, regardless."@@ -432,7 +429,7 @@"Use the index table to search for records."[search-term & [search-ctx]](map (juxt :model :name)- (search.db/rows (search-query search-term search-ctx [:model :name]))))+ (search.db/search-index-rows (search-query search-term search-ctx [:model :name]))))(defn reset-index!"Ensure we have a blank slate; in case the table schema or stored data format has changed."diff --git a/src/metabase/search/appdb/scoring.clj b/src/metabase/search/appdb/scoring.cljindex cf64abaf7b1..d648bd99fad 100644--- a/src/metabase/search/appdb/scoring.clj+++ b/src/metabase/search/appdb/scoring.clj@@ -18,9 +18,7 @@(defn- view-count-percentiles*[p-value]- (into {} (for [{:keys [model vcp]} (search.db/rows (specialization/view-count-percentile-query- (search.index/active-table)- p-value))]+ (into {} (for [{:keys [model vcp]} (search.db/view-count-percentile-rows (search.index/active-table) p-value)][(keyword model) vcp])))(def ^{:private truediff --git a/src/metabase/search/appdb/specialization/postgres.clj b/src/metabase/search/appdb/specialization/postgres.cljindex 71abfdc83ac..ed438b93d28 100644--- a/src/metabase/search/appdb/specialization/postgres.clj+++ b/src/metabase/search/appdb/specialization/postgres.clj@@ -25,18 +25,7 @@"CREATE INDEX IF NOT EXISTS %s_archived_idx ON %s (archived)"]))(defmethod specialization/batch-upsert! :postgres [table entries]- (when (seq entries)- (search.db/execute!- ;; The cost of dynamically calculating these keys should be small compared to the IO cost, so unoptimized. Note- ;; that the entries are not guaranteed to be homogeneous - some may be missing nullable columns. So we need to- ;; get the keys from *all* the entries.- (let [update-keys (vec (disj (set (mapcat keys entries)) :id :model :model_id))- excluded-kw (fn [column] (keyword (str "excluded." (name column))))]- {:insert-into table- :values entries- :on-conflict [:model :model_id]- :do-update-set (with-meta (zipmap update-keys (map excluded-kw update-keys))- {:allow-subquery true})}))))+ (search.db/postgres-batch-upsert! table entries))(defmethod specialization/base-query :postgres[active-table search-term search-ctx select-items]@@ -103,7 +92,7 @@(defmethod specialization/analyze-table! :postgres[table-name]- (search.db/execute! (str "ANALYZE " (name table-name))))+ (search.db/analyze-search-index-table! table-name))(defmethod specialization/index-size-estimate :postgres[table-name]diff --git a/src/metabase/search/db.clj b/src/metabase/search/db.cljindex cb0c9ae6289..c6bbc16f97e 100644--- a/src/metabase/search/db.clj+++ b/src/metabase/search/db.clj@@ -2,18 +2,82 @@"Application database queries for the search module. Every function here is a direct Toucan 2 call with noadditional logic, so no other namespace in the module runs a query itself (connection and transaction handling still use `toucan2.core`)."(:require+ [honey.sql.helpers :as sql.helpers]+ [metabase.search.appdb.specialization.api :as specialization][toucan2.core :as t2]))-(defn rows- "The rows returned by the Honey SQL or raw SQL `query`."+(defn spec-index-rows+ "The rows matching the Honey SQL `query` built from a search model's spec by `metabase.search.ingestion`."[query](t2/query query))-(defn execute!- "Run the Honey SQL or raw SQL `statement`."+(defn scored-search-rows+ "The rows matching the scored, filtered search Honey SQL `query` built by `metabase.search.appdb.core`."+ [query]+ (t2/query query))++(defn distinct-model-rows+ "The rows matching the Honey SQL `query` built by `metabase.search.appdb.core` to find the distinct search models+ present in the results."+ [query]+ (t2/query query))++(defn search-index-probe-rows+ "The rows matching the Honey SQL `query` built by `metabase.search.appdb.core` to check whether a single row+ survives a filter."+ [query]+ (t2/query query))++(defn search-index-rows+ "The rows matching the Honey SQL `query` built by `metabase.search.appdb.index/search-query`."+ [query]+ (t2/query query))++(defn view-count-percentile-rows+ "The Model to view-count-percentile rows for the search index table `index-table` at percentile `p-value`."+ [index-table p-value]+ (t2/query (specialization/view-count-percentile-query index-table p-value)))++(defn drop-search-index-table-if-exists!+ "Drop the search index table named `table-name`, if it exists."+ [table-name]+ (t2/query (sql.helpers/drop-table :if-exists table-name)))++(defn drop-search-index-table!+ "Drop the search index table named `table-name`."+ [table-name]+ (t2/query (sql.helpers/drop-table table-name)))++(defn create-search-index-table!+ "Create the search index table named `table-name` with `columns` (the Honey SQL column definitions built by the+ active search engine specialization)."+ [table-name columns]+ (t2/query (-> (sql.helpers/create-table table-name)+ (sql.helpers/with-columns columns))))++(defn run-search-index-statement!+ "Run a single post-creation SQL statement (e.g. an index creation) for a search index table."[statement](t2/query statement))+(defn analyze-search-index-table!+ "Run `ANALYZE` on the search index table `table-name` (Postgres only)."+ [table-name]+ (t2/query (str "ANALYZE " (name table-name))))++(defn postgres-batch-upsert!+ "Upsert `entries` into the search index `table`, on conflict of `(model, model_id)` overwriting every other column+ with the new value."+ [table entries]+ (when (seq entries)+ (let [update-keys (vec (disj (set (mapcat keys entries)) :id :model :model_id))+ excluded-kw (fn [column] (keyword (str "excluded." (name column))))]+ (t2/query {:insert-into table+ :values entries+ :on-conflict [:model :model_id]+ :do-update-set (with-meta (zipmap update-keys (map excluded-kw update-keys))+ {:allow-subquery true})}))))+(defn commit!"Commit the current transaction."[]diff --git a/src/metabase/search/ingestion.clj b/src/metabase/search/ingestion.cljindex 308e7e42f44..8d5c5d94c95 100644--- a/src/metabase/search/ingestion.clj+++ b/src/metabase/search/ingestion.clj@@ -224,7 +224,7 @@(let [spec (search.spec/spec search-model)indexed? (-> (spec-index-query-where search-model [:= :this.id id])(assoc :select [[[:inline 1] :one]] :limit 1)- search.db/rows+ search.db/spec-index-rowsseqboolean)](cond@@ -276,7 +276,7 @@(->> (for [model search.spec/search-models](-> (spec-index-query-where model nil)(assoc :select [[:%count.* :count]])- search.db/rows+ search.db/spec-index-rowsfirst:count))(filter some?)diff --git a/src/metabase/session/db.clj b/src/metabase/session/db.cljindex 3d0abb3df0b..ae00191c010 100644--- a/src/metabase/session/db.clj+++ b/src/metabase/session/db.clj@@ -3,6 +3,7 @@additional logic, so no other namespace in the module runs a query itself (model definitions still use `toucan2.core`)."(:require[metabase.app-db.core :as mdb]+ [metabase.auth-identity.db :as auth-identity.db][metabase.tracing.core :as tracing][metabase.util :as u][metabase.util.honey-sql-2 :as h2x]@@ -37,14 +38,15 @@(t2/query-one hsql))))(defn auth-identity-for-provider- "The AuthIdentity of the User with `user-id` at `provider`, or nil."+ "The AuthIdentity of the User with `user-id` at `provider`, or nil. See `metabase.auth-identity.db/auth-identity`,+ which owns the AuthIdentity table."[user-id provider]- (t2/select-one :model/AuthIdentity :user_id user-id :provider provider))+ (auth-identity.db/auth-identity user-id provider))(defn auth-identity-exists?"Whether the User with `user-id` has an AuthIdentity at `provider`."[user-id provider]- (t2/exists? :model/AuthIdentity :user_id user-id :provider provider))+ (auth-identity.db/auth-identity-exists? user-id provider))(defn set-auth-identity-credentials!"Set the `credentials` of the AuthIdentity with `auth-identity-id`."diff --git a/src/metabase/sync/analyze/fingerprint.clj b/src/metabase/sync/analyze/fingerprint.cljindex c15d67b893c..27dc192d267 100644--- a/src/metabase/sync/analyze/fingerprint.clj+++ b/src/metabase/sync/analyze/fingerprint.clj@@ -2,10 +2,7 @@"Analysis sub-step that takes a sample of values for a Field and saving a non-identifying fingerprintused for classification. This fingerprint is saved as a column on the Field it belongs to."(:require- [clojure.set :as set]- [honey.sql.helpers :as sql.helpers][metabase.analyze.core :as analyze]- [metabase.app-db.core :as app-db][metabase.driver :as driver][metabase.driver.util :as driver.u][metabase.lib.schema.metadata.fingerprint :as lib.schema.metadata.fingerprint]@@ -101,98 +98,11 @@;;; | WHICH FIELDS NEED UPDATED FINGERPRINTS? |;;; +----------------------------------------------------------------------------------------------------------------+-;; Logic for building the somewhat-complicated query we use to determine which Fields need new Fingerprints-;;-;; This ends up giving us a SQL query that looks something like:-;;-;; SELECT *-;; FROM metabase_field-;; WHERE active = true-;; AND (semantic_type NOT IN ('type/PK') OR semantic_type IS NULL)-;; AND preview_display = true-;; AND visibility_type <> 'retired'-;; AND table_id = 1-;; AND ((fingerprint_version < 1 AND-;; base_type IN ("type/Longitude", "type/Latitude", "type/Integer"))-;; OR-;; (fingerprint_version < 2 AND-;; base_type IN ("type/Text", "type/SerializedJSON")))--(mu/defn- base-types->descendants :- [:maybe [:set ms/FieldTypeKeywordOrString]]- "Given a set of `base-types` return an expanded set that includes those base types as well as all of their- descendants. These types are converted to strings so HoneySQL doesn't confuse them for columns."- [base-types :- [:set ms/FieldType]]- (into #{}- (comp (mapcat (fn [base-type]- (cons base-type (descendants base-type))))- (map u/qualified-name))- base-types))--;; It's even cooler if we could generate efficient SQL that looks at what types have already-;; been marked for upgrade so we don't need to generate overly-complicated queries.-;;-;; e.g. instead of doing:-;;-;; WHERE ((version < 2 AND base_type IN ("type/Integer", "type/BigInteger", "type/Text")) OR-;; (version < 1 AND base_type IN ("type/Boolean", "type/Integer", "type/BigInteger")))-;;-;; we could do:-;;-;; WHERE ((version < 2 AND base_type IN ("type/Integer", "type/BigInteger", "type/Text")) OR-;; (version < 1 AND base_type IN ("type/Boolean")))-;;-;; (In the example above, something that is a `type/Integer` or `type/Text` would get upgraded-;; as long as it's less than version 2; so no need to also check if those types are less than 1, which-;; would always be the case.)-;;-;; This way we can also completely omit adding clauses for versions that have been "eclipsed" by others.-;; This would keep the SQL query from growing boundlessly as new fingerprint versions are added-(mu/defn- versions-clauses :- [:maybe [:sequential :any]]- []- ;; keep track of all the base types (including descendants) for each version, starting from most recent- (let [versions+base-types (reverse (sort-by first (seq i/*fingerprint-version->types-that-should-be-re-fingerprinted*)))- already-seen (atom #{})]- (for [[version base-types] versions+base-types- :let [descendants (base-types->descendants base-types)- not-yet-seen (set/difference descendants @already-seen)]- ;; if all the descendants of any given version have already been seen, we can skip this clause altogether- :when (seq not-yet-seen)]- ;; otherwise record the newly seen types and generate an appropriate clause- (do- (swap! already-seen set/union not-yet-seen)- [:and- [:< :fingerprint_version version]- [:in :base_type not-yet-seen]]))))--(def ^:private fields-to-fingerprint-base-clause- "Base clause to get fields for fingerprinting. When refingerprinting, run as is. When fingerprinting in analysis, only- look for fields without a fingerprint or whose version can be updated. This clauses is added on- by [[versions-clauses]]."- [:and- [:= :active true]- [:or- [:not (app-db/isa :semantic_type :type/PK)]- [:= :semantic_type nil]]- [:not-in :visibility_type ["retired" "sensitive"]]- [:not-in :base_type (conj (app-db/type-keyword->descendants :type/fingerprint-unsupported)- (u/qualified-name :type/*))]])-(def ^:dynamic *refingerprint?*"Whether we are refingerprinting or doing the normal fingerprinting. Refingerprinting should get fields that alreadyare analyzed and have fingerprints."false)-(mu/defn- honeysql-for-fields-that-need-fingerprint-updating :- [:map- [:where :any]]- "Return appropriate WHERE clause for all the Fields whose Fingerprint needs to be re-calculated."- ([]- {:where (cond-> fields-to-fingerprint-base-clause- (not *refingerprint?*) (conj (cons :or (versions-clauses))))})-- ([table :- i/TableInstance]- (sql.helpers/where (honeysql-for-fields-that-need-fingerprint-updating)- [:= :table_id (u/the-id table)])))-;;; +----------------------------------------------------------------------------------------------------------------+;;; | FINGERPRINTING ALL FIELDS IN A TABLE |;;; +----------------------------------------------------------------------------------------------------------------+@@ -202,7 +112,11 @@id so the selection is stable across syncs. This should include NEW fields that are active and visible."[table :- i/TableInstancelimit :- ms/PositiveInt]- (seq (sync.db/fields-where (:where (honeysql-for-fields-that-need-fingerprint-updating table)) limit)))+ (seq (sync.db/fields-needing-fingerprint-update+ (u/the-id table)+ *refingerprint?*+ i/*fingerprint-version->types-that-should-be-re-fingerprinted*+ limit)))(mu/defn- warn-too-many-fields!"Log that `table` has more fields to fingerprint than fingerprint-max-fields-per-table (`limit`), so only the firstdiff --git a/src/metabase/sync/db.clj b/src/metabase/sync/db.cljindex 43622ed757d..08533d0159c 100644--- a/src/metabase/sync/db.clj+++ b/src/metabase/sync/db.clj@@ -2,6 +2,11 @@"Application database queries for the sync module. Every function here is a direct Toucan 2 call with no additionallogic, so the rest of the module never talks to `toucan2.core` itself."(:require+ [clojure.set :as set]+ [honey.sql :as sql]+ [metabase.app-db.core :as app-db]+ [metabase.util :as u]+ [metabase.util.honey-sql-2 :as h2x][metabase.warehouse-schema.models.table :as table][toucan2.core :as t2]))@@ -216,10 +221,61 @@:table_id :has_field_values]:id [:in field-ids]))-(defn fields-where- "Up to `limit` Fields matching the Honey SQL `where`, ordered by ID."- [where limit]- (t2/select :model/Field {:where where, :order-by [[:id :asc]], :limit limit}))+(defn- base-types->descendants+ "Given a set of `base-types`, an expanded set including those types and all their descendants in the type+ hierarchy, as qualified strings so HoneySQL doesn't confuse them for columns."+ [base-types]+ (into #{}+ (comp (mapcat (fn [base-type] (cons base-type (descendants base-type))))+ (map u/qualified-name))+ base-types))++(defn- fingerprint-version-clauses+ "Honey SQL `:or` disjuncts, one per fingerprint version older than the latest that hasn't already been superseded,+ matching Fields whose `base_type` (or a descendant) was re-fingerprinted at that version. `version->base-types` is+ a map of fingerprint version to the set of base types re-fingerprinted at that version."+ [version->base-types]+ (let [versions+base-types (reverse (sort-by first (seq version->base-types)))+ already-seen (atom #{})]+ (into [:or]+ (keep (fn [[version base-types]]+ (let [not-yet-seen (set/difference (base-types->descendants base-types) @already-seen)]+ (when (seq not-yet-seen)+ (swap! already-seen set/union not-yet-seen)+ [:and+ [:< :fingerprint_version version]+ [:in :base_type not-yet-seen]]))))+ versions+base-types)))++(def ^:private fields-to-fingerprint-base-clause+ [:and+ [:= :active true]+ [:or+ [:not (app-db/isa :semantic_type :type/PK)]+ [:= :semantic_type nil]]+ [:not-in :visibility_type ["retired" "sensitive"]]+ [:not-in :base_type (conj (app-db/type-keyword->descendants :type/fingerprint-unsupported)+ (u/qualified-name :type/*))]])++(defn- needs-fingerprint-update-clause+ "Honey SQL clause matching Fields whose fingerprint needs to be re-calculated: active, non-`PK`/no-semantic-type,+ non-retired/sensitive-visibility, non-fingerprint-unsupported `base_type`, and, unless `refingerprint?`, without a+ fingerprint or whose fingerprint version can be updated per `version->base-types` (a map of fingerprint version to+ the set of base types that should be re-fingerprinted at that version)."+ [refingerprint? version->base-types]+ (cond-> fields-to-fingerprint-base-clause+ (not refingerprint?) (conj (fingerprint-version-clauses version->base-types))))++(defn fields-needing-fingerprint-update+ "Up to `limit` active, visible Fields of the Table with `table-id` whose fingerprint needs to be re-calculated,+ ordered by ID. See [[needs-fingerprint-update-clause]] for the full matching criteria."+ [table-id refingerprint? version->base-types limit]+ (t2/select :model/Field+ {:where [:and+ [:= :table_id table-id]+ (needs-fingerprint-update-clause refingerprint? version->base-types)]+ :order-by [[:id :asc]]+ :limit limit}))(defn field-fingerprint"The fingerprint of the Field with `field-id`."@@ -345,9 +401,13 @@(t2/update! :model/Field :id [:in field-ids] {:fingerprint_version fingerprint-version}))(defn set-table-fields-indexed!- "Set `database_indexed` of the Fields of the Table with `table-id` to the SQL expression `indexed-expr`."- [table-id indexed-expr]- (t2/update! :model/Field {:table_id table-id} {:database_indexed indexed-expr}))+ "Mark the Fields of the Table with `table-id` whose id is in `indexed-field-ids` as indexed, and all its other+ Fields as not indexed."+ [table-id indexed-field-ids]+ (t2/update! :model/Field {:table_id table-id}+ {:database_indexed (if (seq indexed-field-ids)+ [:case [:in :id indexed-field-ids] true :else false]+ false)}))(defn set-top-level-fields-indexed!"Set `database_indexed` of the top-level Fields with `field-ids` to `indexed?`."@@ -375,10 +435,83 @@:where [:and sync-tables-clause [:= :db_id database-id]]}]}{:last_analyzed :%now}))-(defn execute-one!- "Run the SQL statement `sql` and return its single result."- [sql]- (t2/query-one sql))+(defn- fk-field-id-subquery+ "Subquery selecting the (lowest) id of the Field named `column-name` on the Table `[table-schema table-name]` in+ the Database with `db-id`, excluding Fields with a user-set foreign key target or semantic type. `min` limits the+ subquery to one result (MySQL disallows `LIMIT` in subqueries), needed because schema/table/column names can be+ non-unique when lower-cased for some DBs."+ [db-id table-schema table-name column-name]+ ^:allow-subquery+ {:select [[[:min :f.id] :id]]+ :from [[:metabase_field :f]]+ :join [[:metabase_table :t] [:= :f.table_id :t.id]]+ :left-join [[:metabase_field_user_settings :u] [:= :f.id :u.field_id]]+ :where [:and+ [:= :u.fk_target_field_id nil]+ [:= :u.semantic_type nil]+ [:= :t.db_id db-id]+ [:= [:lower :f.name] (u/lower-case-en column-name)]+ [:= [:lower :t.name] (u/lower-case-en table-name)]+ [:= [:lower :t.schema] (some-> table-schema u/lower-case-en)]+ [:= :f.active true]+ [:not= :f.visibility_type "retired"]+ [:= :t.active true]+ [:= :t.visibility_type nil]]})++(defn- fk-target-changed-clause+ "Honey SQL clause true when the Field's `fk_target_field_id` is not already `pk-id-expr`."+ [pk-id-expr]+ [:or+ [:= :f.fk_target_field_id nil]+ [:not= :f.fk_target_field_id pk-id-expr]])++(defn- mark-fk-statement+ "`[sql & params]` updating the `fk_target_field_id` of the Field at `[fk-table-schema fk-table-name+ fk-column-name]` in the Database with `db-id` to the id of the Field at `[pk-table-schema pk-table-name+ pk-column-name]`, unless it already points there, per the application DB's dialect."+ [db-id fk-table-schema fk-table-name fk-column-name pk-table-schema pk-table-name pk-column-name]+ (let [fk-field-id-query (fk-field-id-subquery db-id fk-table-schema fk-table-name fk-column-name)+ pk-field-id-query (fk-field-id-subquery db-id pk-table-schema pk-table-name pk-column-name)+ q (case (app-db/db-type)+ :mysql+ {:update [:metabase_field :f]+ :join [[fk-field-id-query :fk] [:= :fk.id :f.id]+ [pk-field-id-query :pk]+ (fk-target-changed-clause :pk.id)]+ :set {:fk_target_field_id :pk.id+ ;; We need to reset has_field_values when it is auto-list as FKs should not be marked as such+ :has_field_values [:case [:= :has_field_values "auto-list"] nil :else :has_field_values]+ :semantic_type "type/FK"}}+ :postgres+ {:update [:metabase_field :f]+ :from [[fk-field-id-query :fk]]+ :join [[pk-field-id-query :pk] true]+ :set {:fk_target_field_id :pk.id+ ;; We need to reset has_field_values when it is auto-list as FKs should not be marked as such+ :has_field_values [:case [:= :has_field_values "auto-list"] nil :else :has_field_values]+ :semantic_type "type/FK"}+ :where [:and+ [:= :fk.id :f.id]+ (fk-target-changed-clause :pk.id)]}+ :h2+ {:update [:metabase_field :f]+ :set {:fk_target_field_id pk-field-id-query+ ;; We need to reset has_field_values when it is auto-list as FKs should not be marked as such+ :has_field_values [:case [:= :has_field_values "auto-list"] nil :else :has_field_values]+ :semantic_type "type/FK"}+ :where [:and+ [:= :f.id fk-field-id-query]+ [:not= pk-field-id-query nil]+ (fk-target-changed-clause pk-field-id-query)]})]+ (sql/format q :dialect (app-db/quoting-style (app-db/db-type)))))++(defn mark-fk!+ "Set the `fk_target_field_id` of the Field at `[fk-table-schema fk-table-name fk-column-name]` in the Database with+ `db-id` to the id of the Field at `[pk-table-schema pk-table-name pk-column-name]`, unless it already points there.+ Returns 1 if a Field was updated, 0 otherwise."+ [db-id fk-table-schema fk-table-name fk-column-name pk-table-schema pk-table-name pk-column-name]+ (t2/query-one (mark-fk-statement db-id fk-table-schema fk-table-name fk-column-name+ pk-table-schema pk-table-name pk-column-name)));;; ---------------------------------------------- FieldValues ----------------------------------------------@@ -387,13 +520,19 @@[field-id](t2/exists? :model/FieldValues :field_id field-id))+(defn- before-max-age-value+ "Honey SQL `[:< …]` value expression matching a timestamp more than `max-age-days` days before now."+ [max-age-days]+ [:< (h2x/add-interval-honeysql-form (app-db/db-type) :%now (- max-age-days) :day)])+(defn advanced-field-values-count-before- "The number of FieldValues of `types` for the Field with `field-id` created before the SQL expression- `created-before`."- [field-id types created-before]- (t2/count :model/FieldValues :field_id field-id :type [:in types] :created_at [:< created-before]))+ "The number of FieldValues of `types` for the Field with `field-id` created more than `max-age-days` days ago."+ [field-id types max-age-days]+ (t2/count :model/FieldValues :field_id field-id :type [:in types]+ :created_at (before-max-age-value max-age-days)))(defn delete-advanced-field-values-before!- "Delete the FieldValues of `types` for the Field with `field-id` created before the SQL expression `created-before`."- [field-id types created-before]- (t2/delete! :model/FieldValues :field_id field-id :type [:in types] :created_at [:< created-before]))+ "Delete the FieldValues of `types` for the Field with `field-id` created more than `max-age-days` days ago."+ [field-id types max-age-days]+ (t2/delete! :model/FieldValues :field_id field-id :type [:in types]+ :created_at (before-max-age-value max-age-days)))diff --git a/src/metabase/sync/field_values.clj b/src/metabase/sync/field_values.cljindex 88148a61975..80533dfc333 100644--- a/src/metabase/sync/field_values.clj+++ b/src/metabase/sync/field_values.clj@@ -2,7 +2,6 @@"Logic for updating FieldValues for fields in a database."(:require[java-time.api :as t]- [metabase.app-db.core :as mdb][metabase.driver :as driver][metabase.driver.util :as driver.u][metabase.sync.db :as sync.db]@@ -268,14 +267,10 @@(defn- delete-expired-advanced-field-values-for-field)- (let [types field-values/advanced-field-values-types- cutoff ((requiring-resolve 'metabase.util.honey-sql-2/add-interval-honeysql-form)- (mdb/db-type)- :%now- (- (t/as field-values/advanced-field-values-max-age :days))- :day)- rows-count (sync.db/advanced-field-values-count-before (:id field) types cutoff)]- (sync.db/delete-advanced-field-values-before! (:id field) types cutoff)+ (let [types field-values/advanced-field-values-types+ max-age-days (t/as field-values/advanced-field-values-max-age :days)+ rows-count (sync.db/advanced-field-values-count-before (:id field) types max-age-days)]+ (sync.db/delete-advanced-field-values-before! (:id field) types max-age-days)rows-count)))(mu/defn delete-expired-advanced-field-values-for-table!diff --git a/src/metabase/sync/sync_metadata/fks.clj b/src/metabase/sync/sync_metadata/fks.cljindex b337082ac05..8556bbf3665 100644--- a/src/metabase/sync/sync_metadata/fks.clj+++ b/src/metabase/sync/sync_metadata/fks.clj@@ -1,8 +1,6 @@(ns metabase.sync.sync-metadata.fks"Logic for updating FK properties of Fields from metadata fetched from a physical DB."(:require- [honey.sql :as sql]- [metabase.app-db.core :as mdb][metabase.driver.util :as driver.u][metabase.sync.db :as sync.db][metabase.sync.fetch-metadata :as fetch-metadata]@@ -13,87 +11,13 @@[metabase.util.malli :as mu][metabase.warehouse-schema.models.table :as table]))-(defn ^:private mark-fk-sql- "Returns [sql & params] for [[mark-fk!]] according to the application DB's dialect."- [db-id {:keys [fk-table-name- fk-table-schema- fk-column-name- pk-table-name- pk-table-schema- pk-column-name]}]- (let [field-id-query (fn [db-id table-schema table-name column-name]- ^:allow-subquery- {:select [[[:min :f.id] :id]]- ;; Cal 2024-03-04: We use `min` to limit this subquery to one result (limit 1 isn't allowed- ;; in subqueries in MySQL) because it's possible for schema, table, or column names to be- ;; non-unique when lower-cased for some DBs. We have been doing case-insensitive matching- ;; since #5510 so this preserves behaviour to avoid possible regressions.- ;; It's possible this is to avoid- :from [[:metabase_field :f]]- :join [[:metabase_table :t] [:= :f.table_id :t.id]]- :left-join [[:metabase_field_user_settings :u] [:= :f.id :u.field_id]]- :where [:and- ;; ensure we are not overriding user-set fks- [:= :u.fk_target_field_id nil]- [:= :u.semantic_type nil]- [:= :t.db_id db-id]- [:= [:lower :f.name] (u/lower-case-en column-name)]- [:= [:lower :t.name] (u/lower-case-en table-name)]- [:= [:lower :t.schema] (some-> table-schema u/lower-case-en)]- [:= :f.active true]- [:not= :f.visibility_type "retired"]- [:= :t.active true]- [:= :t.visibility_type nil]]})- fk-field-id-query (field-id-query db-id fk-table-schema fk-table-name fk-column-name)- pk-field-id-query (field-id-query db-id pk-table-schema pk-table-name pk-column-name)-- ;; Only update if either:- ;; - fk_target_field_id is NULL and the new target is not NULL- ;; - fk_target_field_id is not NULL but the new target is different and not NULL- valid-condition- (fn [k]- [:or- [:= :f.fk_target_field_id nil]- [:not= :f.fk_target_field_id k]])-- q (case (mdb/db-type)- :mysql- {:update [:metabase_field :f]- :join [[fk-field-id-query :fk] [:= :fk.id :f.id]- [pk-field-id-query :pk]- (valid-condition :pk.id)]- :set {:fk_target_field_id :pk.id- ;; We need to reset has_field_values when it is auto-list as FKs should not be marked as such- :has_field_values [:case [:= :has_field_values "auto-list"] nil :else :has_field_values]- :semantic_type "type/FK"}}- :postgres- {:update [:metabase_field :f]- :from [[fk-field-id-query :fk]]- :join [[pk-field-id-query :pk] true]- :set {:fk_target_field_id :pk.id- ;; We need to reset has_field_values when it is auto-list as FKs should not be marked as such- :has_field_values [:case [:= :has_field_values "auto-list"] nil :else :has_field_values]- :semantic_type "type/FK"}- :where [:and- [:= :fk.id :f.id]- (valid-condition :pk.id)]}- :h2- {:update [:metabase_field :f]- :set {:fk_target_field_id pk-field-id-query- ;; We need to reset has_field_values when it is auto-list as FKs should not be marked as such- :has_field_values [:case [:= :has_field_values "auto-list"] nil :else :has_field_values]- :semantic_type "type/FK"}- :where [:and- [:= :f.id fk-field-id-query]- [:not= pk-field-id-query nil]- (valid-condition pk-field-id-query)]})]- (sql/format q :dialect (mdb/quoting-style (mdb/db-type)))))-(mu/defn- mark-fk!"Updates the `fk_target_field_id` of a Field. Returns 1 if the Field was successfully updated, 0 otherwise."[database :- i/DatabaseInstancemetadata :- i/FKMetadataEntry]- (u/prog1 (sync.db/execute-one! (mark-fk-sql (:id database) metadata))+ (u/prog1 (sync.db/mark-fk! (:id database)+ (:fk-table-schema metadata) (:fk-table-name metadata) (:fk-column-name metadata)+ (:pk-table-schema metadata) (:pk-table-name metadata) (:pk-column-name metadata))(when (= <> 1)(log/info (u/format-color 'cyan "Marking foreign key from %s %s -> %s %s"(sync-util/table-name-for-logging :name (:fk-table-name metadata)diff --git a/src/metabase/sync/sync_metadata/indexes.clj b/src/metabase/sync/sync_metadata/indexes.cljindex 466411f8519..7cd2da73ae7 100644--- a/src/metabase/sync/sync_metadata/indexes.clj+++ b/src/metabase/sync/sync_metadata/indexes.clj@@ -38,10 +38,7 @@(doseq [field-id adding](log/infof "Marking Field %d as indexed" field-id))(if (or (seq adding) (seq removing))- (do (sync.db/set-table-fields-indexed! (:id table)- (if (seq indexed-field-ids)- [:case [:in :id indexed-field-ids] true :else false]- false))+ (do (sync.db/set-table-fields-indexed! (:id table) indexed-field-ids){:total-indexes (count indexed-field-ids):added-indexes (count adding):removed-indexes (count removing)})diff --git a/src/metabase/typed_schemas/db.clj b/src/metabase/typed_schemas/db.cljindex 8c88db336b9..71a15fc597a 100644--- a/src/metabase/typed_schemas/db.clj+++ b/src/metabase/typed_schemas/db.clj@@ -2,6 +2,7 @@"Application database queries for the typed schemas module. Every function here is a direct Toucan 2 call with noadditional logic, so the rest of the module never talks to `toucan2.core` itself."(:require+ [metabase.collections.models.collection :as collection][toucan2.core :as t2]))(defn destination-database-ids@@ -9,10 +10,28 @@[database-ids](t2/select-fn-set :id :model/Database :id [:in database-ids] :router_database_id [:not= nil]))+(defn- scope-filter-clause+ "Compiles a resolved scope into a Honey SQL where-clause conjunct: a nil scope means unscoped (no clause), an+ empty scope matches nothing."+ [scope-ids column]+ (when scope-ids+ (if (seq scope-ids)+ [:in column scope-ids]+ ;; no row has id -1: a resolved-but-empty scope matches no rows+ [:= column -1])))+(defn cards-ordered-by-name- "The Cards matching the Honey SQL `where` clause, in name then id order."- [where]- (t2/select :model/Card {:where where, :order-by [[:name :asc] [:id :asc]]}))+ "The readable, non-archived Cards of `card-type` among `database-ids` and/or `collection-ids` (either nil for+ unscoped), in name then id order."+ [card-type database-ids collection-ids]+ (t2/select :model/Card+ {:where [:and+ [:= :type (name card-type)]+ [:= :archived false]+ (collection/visible-collection-filter-clause :collection_id)+ (scope-filter-clause database-ids :database_id)+ (scope-filter-clause collection-ids :collection_id)]+ :order-by [[:name :asc] [:id :asc]]}))(defn field-ids-and-table-ids"The id and Table id of the Fields with `field-ids`."@@ -42,16 +61,6 @@[field-id](t2/select-one-fn :table_id :model/Field :id field-id))-(defn- scope-filter-clause- "Compiles a resolved scope into a Honey SQL where-clause conjunct: a nil scope means unscoped (no clause), an- empty scope matches nothing."- [scope-ids column]- (when scope-ids- (if (seq scope-ids)- [:in column scope-ids]- ;; no row has id -1: a resolved-but-empty scope matches no rows- [:= column -1])))-(defn active-tables-in-scope"The active Tables among `database-ids` and/or `table-ids` (either nil for unscoped), in name then id order."[database-ids table-ids]diff --git a/src/metabase/typed_schemas/schema/common.clj b/src/metabase/typed_schemas/schema/common.cljindex 1fb0c4b4666..105fcea3cb8 100644--- a/src/metabase/typed_schemas/schema/common.clj+++ b/src/metabase/typed_schemas/schema/common.clj@@ -2,7 +2,6 @@"Shared typed-schema source helpers."(:require[medley.core :as m]- [metabase.collections.models.collection :as collection][metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.metabot.core :as metabot]@@ -11,23 +10,6 @@(set! *warn-on-reflection* true)-(defn scope-filter-clause- "Compiles a resolved scope (see [[metabase.typed-schemas.scope]]) into a- Honey SQL where-clause conjunct: a nil scope means unscoped (no clause), an- empty scope matches nothing.-- This is the one place the nil-vs-empty semantics of scopes are enforced when- querying. Do not replace call sites with `(seq ...)` guards — that would- turn \"resolved to nothing\" into \"unscoped\" and select everything.- Compiling the empty scope to an always-false clause (rather than skipping- the query) keeps queries with several scope filters composable."- [scope-ids column]- (when scope-ids- (if (seq scope-ids)- [:in column scope-ids]- ;; no row has id -1: a resolved-but-empty scope matches no rows- [:= column -1])))-(defn destination-db-ids"Returns the subset of `db-ids` that back a destination (routed) database.@@ -45,13 +27,7 @@the same visibility, archived, database and collection filters, and exclude cards backed by adestination (routed) database -- see [[destination-db-ids]]."[card-type database-ids collection-ids]- (let [cards (->> (typed-schemas.db/cards-ordered-by-name- (cond-> [:and- [:= :type (name card-type)]- [:= :archived false]- (collection/visible-collection-filter-clause :collection_id)]- database-ids (conj (scope-filter-clause database-ids :database_id))- collection-ids (conj (scope-filter-clause collection-ids :collection_id))))+ (let [cards (->> (typed-schemas.db/cards-ordered-by-name card-type database-ids collection-ids)(filter mi/can-read?))destination-ids (destination-db-ids (into #{} (keep :database_id) cards))](if (seq destination-ids)diff --git a/src/metabase/warehouse_schema/db.clj b/src/metabase/warehouse_schema/db.cljindex dc0f6395471..8de4de6e3ab 100644--- a/src/metabase/warehouse_schema/db.clj+++ b/src/metabase/warehouse_schema/db.clj@@ -4,6 +4,7 @@methods, and transactions."(:require[metabase.app-db.core :as app-db]+ [metabase.models.db :as models.db][toucan2.core :as t2]))(def field-order-rule@@ -17,22 +18,11 @@[field-id](t2/select-one :model/Field :id field-id))-(defn- field-in-path-query- [table-id [field & rest]]- (when field- ^:allow-subquery {:from [:metabase_field]- :select [:id]- :where [:and- [:= :table_id table-id]- [:= :name field]- [:= :parent_id (field-in-path-query table-id rest)]]}))-(defn field-in-path"The Field named by the last of `field-names` (each nested inside the previous, bottom-most first) under- `table-id`, or nil."+ `table-id`, or nil. See `metabase.models.db/field-in-path`, which owns the shared query."[table-id field-names]- (when (seq field-names)- (t2/select-one :model/Field (field-in-path-query table-id field-names))))+ (models.db/field-in-path table-id field-names))(defn fields"The Fields with `field-ids`."diff --git a/src/metabase/warehouses_rest/api.clj b/src/metabase/warehouses_rest/api.cljindex f139c6a4b38..59410948a09 100644--- a/src/metabase/warehouses_rest/api.clj+++ b/src/metabase/warehouses_rest/api.clj@@ -8,7 +8,6 @@[metabase.api.macros :as api.macros][metabase.app-db.core :as mdb][metabase.classloader.core :as classloader]- [metabase.collections.models.collection :as collection][metabase.config.core :as config][metabase.database-routing.core :as database-routing][metabase.driver :as driver]@@ -205,7 +204,7 @@(mu/defn- source-query-cards"Fetch the Cards that can be used as source queries (e.g. presented as virtual tables)."[card-type :- ::queries.schema/card-type- & {:keys [additional-constraints xform], :or {xform identity}}]+ & {:keys [collection-scope xform], :or {xform identity}}](when-let [ids-of-dbs-that-support-source-queries (not-empty (ids-of-dbs-that-support-source-queries))](transduce(comp (map (partial mi/do-after-select :model/Card))@@ -214,14 +213,7 @@(completing conj #(t2/hydrate % :collection :metrics))[](warehouses-rest.db/source-query-cards-reducible- (into [:and- [:not= :result_metadata nil]- [:= :archived false]- ;; always return metrics for now- [:in :type [(u/qualified-name card-type) "metric"]]- [:in :database_id ids-of-dbs-that-support-source-queries]- (collection/visible-collection-filter-clause)]- additional-constraints)))))+ card-type ids-of-dbs-that-support-source-queries collection-scope))))(mu/defn- source-query-cards-exist?"Truthy if a single Card that can be used as a source query exists."@@ -316,19 +308,8 @@user-info {:user-id api/*current-user-id*:is-superuser? (mi/superuser?):is-data-analyst? api/*is-data-analyst?*}- base-where [:and- [:= :is_stub false]- (when-not include-analytics?- [:= :is_audit false])- (if filter-on-router-database-id- [:= :router_database_id router-database-id]- [:= :router_database_id nil])]- where-clause (if filter-by-data-access?- [:and base-where [:or (:clause (mi/visible-filter-clause :model/Database :id user-info {:perms/create-queries :query-builder}))- (:clause (mi/visible-filter-clause :model/Database :id user-info {:perms/manage-database :yes}))- (:clause (mi/visible-filter-clause :model/Database :id user-info {:perms/manage-table-metadata :yes}))]]- base-where)- dbs (warehouses-rest.db/databases-where where-clause)+ dbs (warehouses-rest.db/databases-where user-info filter-by-data-access? filter-on-router-database-id+ include-analytics?);; everything below walks the list one database at a time_ (perms/prime-database-perms-cache {:db-ids (into #{} (map :id) dbs)})](cond-> (-> dbs add-native-perms-info add-transforms-perms-info)@@ -1326,10 +1307,6 @@(map :schema (f (map (fn [s] {:db_id id :schema s}) schemas)))schemas)(filter (partial can-read-schema? id) schemas)))- clauses (cond-> []- ;; a non-nil value means Table is hidden --- ;; see [[metabase.warehouse-schema.models.table/visibility-types]]- (not include-hidden?) (conj [:= :visibility_type nil]));; For can-query? and can-write-metadata?, we need to filter based on tables in each schemafilter-schemas-by-tables (fn [schemas](if (or can-query? can-write-metadata?)@@ -1342,11 +1319,7 @@(filter #(contains? allowed-schemas %) schemas))schemas))](warehouses/get-database id {:include-editable-data-model? include-editable-data-model?})- (->> (warehouses-rest.db/active-table-schemas id- (merge- {:order-by [[:%lower.schema :asc]]}- (when clauses- {:where (into [:and] clauses)})))+ (->> (warehouses-rest.db/active-table-schemas id include-hidden?)filter-schemasfilter-schemas-by-tables;; for `nil` schemas return the empty string@@ -1520,9 +1493,9 @@(when (lib-be/enable-nested-queries)(->> (source-query-cards:question- :additional-constraints [(if (= schema (schema.table/root-collection-schema-name))- [:= :collection_id nil]- [:in :collection_id (api/check-404 (not-empty (warehouses-rest.db/collection-ids-named schema)))])])+ :collection-scope (if (= schema (schema.table/root-collection-schema-name))+ :root+ (api/check-404 (not-empty (warehouses-rest.db/collection-ids-named schema)))))(map schema.table/card->virtual-table))));; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to@@ -1557,9 +1530,9 @@(when (lib-be/enable-nested-queries)(->> (source-query-cards:model- :additional-constraints [(if (= schema (schema.table/root-collection-schema-name))- [:= :collection_id nil]- [:in :collection_id (api/check-404 (not-empty (warehouses-rest.db/collection-ids-named schema)))])])+ :collection-scope (if (= schema (schema.table/root-collection-schema-name))+ :root+ (api/check-404 (not-empty (warehouses-rest.db/collection-ids-named schema)))))(map schema.table/card->virtual-table))));;; -------------------------------- GET /api/database/:id/settings-available ------------------------------------diff --git a/src/metabase/warehouses_rest/db.clj b/src/metabase/warehouses_rest/db.cljindex 02defdc708f..d8f2fefbf23 100644--- a/src/metabase/warehouses_rest/db.clj+++ b/src/metabase/warehouses_rest/db.clj@@ -4,6 +4,8 @@(:require[clojure.string :as str][metabase.app-db.core :as mdb]+ [metabase.collections.models.collection :as collection]+ [metabase.models.interface :as mi][metabase.util :as u][metabase.util.honey-sql-2 :as h2x][toucan2.core :as t2]))@@ -34,9 +36,11 @@(t2/select [:model/Database :id :engine]))(defn source-query-cards-reducible- "A reducible of the Cards, with their moderation status, matching the Honey SQL `where` clause, in- case-insensitive name order."- [where]+ "A reducible of the Cards of `card-type` (also including \"metric\" Cards) in the Databases with `database-ids`+ visible to the current user that can be used as source queries, with their moderation status, in case-insensitive+ name order. `collection-scope` further restricts by collection: `nil` applies no collection restriction, `:root`+ restricts to Cards with no collection, and a collection of ids restricts to Cards in those collections."+ [card-type database-ids collection-scope](t2/reducible-query {:select [:name :description :database_id :dataset_query :id :collection_id:result_metadata :type :source_card_id :card_schema[^:allow-subquery {:select [:status]@@ -49,14 +53,41 @@:limit 1}:moderated_status]]:from [:report_card]- :where where+ :where [:and+ [:not= :result_metadata nil]+ [:= :archived false]+ [:in :type [(u/qualified-name card-type) "metric"]]+ [:in :database_id database-ids]+ (cond+ (nil? collection-scope) nil+ (= collection-scope :root) [:= :collection_id nil]+ :else [:in :collection_id collection-scope])+ (collection/visible-collection-filter-clause)]:order-by [[:%lower.name :asc]]}))(defn databases-where- "The Databases matching the Honey SQL `where` clause, in name then engine order."- [where]- (t2/select :model/Database {:order-by [:%lower.name :%lower.engine]- :where where}))+ "The Databases visible to the user described by `user-info` (a map of `:user-id`/`:is-superuser?`/+ `:is-data-analyst?`), in name then engine order. Excludes stub Databases and, unless `include-analytics?`, the+ audit Database. Restricted to Databases routed from `router-database-id` when given, otherwise to non-routed+ Databases. When `filter-by-data-access?` is true, further restricted to Databases the user can query, manage, or+ edit the metadata of."+ [user-info filter-by-data-access? router-database-id include-analytics?]+ (let [base-where [:and+ [:= :is_stub false]+ (when-not include-analytics?+ [:= :is_audit false])+ (if router-database-id+ [:= :router_database_id router-database-id]+ [:= :router_database_id nil])]+ where (if filter-by-data-access?+ [:and base-where+ [:or+ (:clause (mi/visible-filter-clause :model/Database :id user-info {:perms/create-queries :query-builder}))+ (:clause (mi/visible-filter-clause :model/Database :id user-info {:perms/manage-database :yes}))+ (:clause (mi/visible-filter-clause :model/Database :id user-info {:perms/manage-table-metadata :yes}))]]+ base-where)]+ (t2/select :model/Database {:order-by [:%lower.name :%lower.engine]+ :where where})))(defn database-exists?"Whether a Database with `database-id` exists."@@ -222,9 +253,16 @@(t2/select :model/Table :db_id database-id :active true))(defn active-table-schemas- "The schemas of the active Tables of the Database with `database-id` also selected by the Honey SQL `query`."- [database-id query]- (t2/select-fn-set :schema :model/Table :db_id database-id :active true query))+ "The distinct schemas of the active Tables of the Database with `database-id`, in schema order. When+ `include-hidden?` is false, restricted to Tables with no `visibility_type` (a non-nil value means the Table is+ hidden -- see [[metabase.warehouse-schema.models.table/visibility-types]])."+ [database-id include-hidden?]+ (let [clauses (cond-> []+ (not include-hidden?) (conj [:= :visibility_type nil]))]+ (t2/select-fn-set :schema :model/Table :db_id database-id :active true+ (merge {:order-by [[:%lower.schema :asc]]}+ (when clauses+ {:where (into [:and] clauses)})))))(defn active-tables-in-schema"The active Tables in `schema` of the Database with `database-id`, in display name order."diff --git a/test/metabase/sync/analyze/fingerprint_test.clj b/test/metabase/sync/analyze/fingerprint_test.cljindex 23bf1b596e5..fc811caf911 100644--- a/test/metabase/sync/analyze/fingerprint_test.clj+++ b/test/metabase/sync/analyze/fingerprint_test.clj@@ -6,6 +6,7 @@[metabase.app-db.core :as app-db][metabase.query-processor :as qp][metabase.sync.analyze.fingerprint :as sync.fingerprint]+ [metabase.sync.db :as sync.db][metabase.sync.interface :as i][metabase.test :as mt][metabase.test.data :as data]@@ -23,9 +24,9 @@;; Check that our `base-types->descendants` function properly returns a set of descendants including parent type(deftest ^:parallel base-type->descendats-test(is (= #{"type/URL" "type/ImageURL" "type/AvatarURL"}- (#'sync.fingerprint/base-types->descendants #{:type/URL})))+ (#'sync.db/base-types->descendants #{:type/URL})))(is (= #{"type/ImageURL" "type/AvatarURL"}- (#'sync.fingerprint/base-types->descendants #{:type/ImageURL :type/AvatarURL}))))+ (#'sync.db/base-types->descendants #{:type/ImageURL :type/AvatarURL}))))(def ^:private skip-fingerprint-base-types(into #{"type/*"@@ -44,108 +45,99 @@:type/Large:type/fingerprint-unsupported]))-(deftest ^:parallel honeysql-for-fields-that-need-fingerprint-updating-test- (testing (str "Make sure we generate the correct HoneySQL WHERE clause based on whatever is in "- "`*fingerprint-version->types-that-should-be-re-fingerprinted*`")- (is (= {:where- [:and- [:= :active true]- [:or- [:not (app-db/isa :semantic_type :type/PK)]- [:= :semantic_type nil]]- [:not-in :visibility_type ["retired" "sensitive"]]- [:not-in :base_type skip-fingerprint-base-types]- [:or- [:and- [:< :fingerprint_version 1]- [:in :base_type #{"type/URL" "type/ImageURL" "type/AvatarURL"}]]]]}- (binding [i/*fingerprint-version->types-that-should-be-re-fingerprinted* {1 #{:type/URL}}]- (#'sync.fingerprint/honeysql-for-fields-that-need-fingerprint-updating))))))--(deftest ^:parallel honeysql-for-fields-that-need-fingerprint-updating-test-2- (is (= {:where- [:and- [:= :active true]- [:or- [:not (app-db/isa :semantic_type :type/PK)]- [:= :semantic_type nil]]- [:not-in :visibility_type ["retired" "sensitive"]]- [:not-in :base_type skip-fingerprint-base-types]- [:or- [:and- [:< :fingerprint_version 2]- [:in :base_type #{"type/Decimal" "type/Latitude" "type/Longitude" "type/Coordinate" "type/Currency" "type/Float"- "type/Share" "type/Income" "type/Price" "type/Discount" "type/GrossMargin" "type/Cost" "type/Percentage"}]]- [:and- [:< :fingerprint_version 1]- [:in :base_type #{"type/ImageURL" "type/AvatarURL"}]]]]}- (binding [i/*fingerprint-version->types-that-should-be-re-fingerprinted* {1 #{:type/ImageURL :type/AvatarURL}- 2 #{:type/Float}}]- (#'sync.fingerprint/honeysql-for-fields-that-need-fingerprint-updating)))))--(deftest ^:parallel honeysql-for-fields-that-need-fingerprint-updating-test-3+(deftest ^:parallel needs-fingerprint-update-clause-test+ (testing (str "Make sure we generate the correct HoneySQL WHERE clause based on whatever `version->base-types` "+ "is passed in")+ (is (= [:and+ [:= :active true]+ [:or+ [:not (app-db/isa :semantic_type :type/PK)]+ [:= :semantic_type nil]]+ [:not-in :visibility_type ["retired" "sensitive"]]+ [:not-in :base_type skip-fingerprint-base-types]+ [:or+ [:and+ [:< :fingerprint_version 1]+ [:in :base_type #{"type/URL" "type/ImageURL" "type/AvatarURL"}]]]]+ (#'sync.db/needs-fingerprint-update-clause false {1 #{:type/URL}})))))++(deftest ^:parallel needs-fingerprint-update-clause-test-2+ (is (= [:and+ [:= :active true]+ [:or+ [:not (app-db/isa :semantic_type :type/PK)]+ [:= :semantic_type nil]]+ [:not-in :visibility_type ["retired" "sensitive"]]+ [:not-in :base_type skip-fingerprint-base-types]+ [:or+ [:and+ [:< :fingerprint_version 2]+ [:in :base_type #{"type/Decimal" "type/Latitude" "type/Longitude" "type/Coordinate" "type/Currency" "type/Float"+ "type/Share" "type/Income" "type/Price" "type/Discount" "type/GrossMargin" "type/Cost" "type/Percentage"}]]+ [:and+ [:< :fingerprint_version 1]+ [:in :base_type #{"type/ImageURL" "type/AvatarURL"}]]]]+ (#'sync.db/needs-fingerprint-update-clause false {1 #{:type/ImageURL :type/AvatarURL}+ 2 #{:type/Float}}))))++(deftest ^:parallel needs-fingerprint-update-clause-test-3(testing "our SQL generation code is clever enough to remove version checks when a newer version completely eclipses them"- (is (= {:where- [:and- [:= :active true]- [:or- [:not (app-db/isa :semantic_type :type/PK)]- [:= :semantic_type nil]]- [:not-in :visibility_type ["retired" "sensitive"]]- [:not-in :base_type skip-fingerprint-base-types]- [:or- [:and- [:< :fingerprint_version 2]- [:in :base_type #{"type/Decimal" "type/Latitude" "type/Longitude" "type/Coordinate" "type/Currency" "type/Float"- "type/Share" "type/Income" "type/Price" "type/Discount" "type/GrossMargin" "type/Cost" "type/Percentage"}]]- ;; no type/Float stuff should be included for 1- [:and- [:< :fingerprint_version 1]- [:in :base_type #{"type/URL" "type/ImageURL" "type/AvatarURL"}]]]]}- (binding [i/*fingerprint-version->types-that-should-be-re-fingerprinted* {1 #{:type/Float :type/URL}- 2 #{:type/Float}}]- (#'sync.fingerprint/honeysql-for-fields-that-need-fingerprint-updating))))))--(deftest ^:parallel honeysql-for-fields-that-need-fingerprint-updating-test-4+ (is (= [:and+ [:= :active true]+ [:or+ [:not (app-db/isa :semantic_type :type/PK)]+ [:= :semantic_type nil]]+ [:not-in :visibility_type ["retired" "sensitive"]]+ [:not-in :base_type skip-fingerprint-base-types]+ [:or+ [:and+ [:< :fingerprint_version 2]+ [:in :base_type #{"type/Decimal" "type/Latitude" "type/Longitude" "type/Coordinate" "type/Currency" "type/Float"+ "type/Share" "type/Income" "type/Price" "type/Discount" "type/GrossMargin" "type/Cost" "type/Percentage"}]]+ ;; no type/Float stuff should be included for 1+ [:and+ [:< :fingerprint_version 1]+ [:in :base_type #{"type/URL" "type/ImageURL" "type/AvatarURL"}]]]]+ (#'sync.db/needs-fingerprint-update-clause false {1 #{:type/Float :type/URL}+ 2 #{:type/Float}})))))++(deftest ^:parallel needs-fingerprint-update-clause-test-4(testing "our SQL generation code is also clever enough to completely skip completely eclipsed versions"- (is (= {:where- [:and- [:= :active true]- [:or- [:not (app-db/isa :semantic_type :type/PK)]- [:= :semantic_type nil]]- [:not-in :visibility_type ["retired" "sensitive"]]- [:not-in :base_type skip-fingerprint-base-types]- [:or- [:and- [:< :fingerprint_version 4]- [:in :base_type #{"type/Decimal" "type/Latitude" "type/Longitude" "type/Coordinate" "type/Currency" "type/Float"- "type/Share" "type/Income" "type/Price" "type/Discount" "type/GrossMargin" "type/Cost" "type/Percentage"}]]- [:and- [:< :fingerprint_version 3]- [:in :base_type #{"type/URL" "type/ImageURL" "type/AvatarURL"}]]- ;; version 2 can be eliminated completely since everything relevant there is included in 4- ;; The only things that should go in 1 should be `:type/City` since `:type/Coordinate` is included in 4- [:and- [:< :fingerprint_version 1]- [:in :base_type #{"type/City"}]]]]}- (binding [i/*fingerprint-version->types-that-should-be-re-fingerprinted* {1 #{:type/Coordinate :type/City}- 2 #{:type/Coordinate}- 3 #{:type/URL}- 4 #{:type/Float}}]- (#'sync.fingerprint/honeysql-for-fields-that-need-fingerprint-updating))))))--(deftest ^:parallel honeysql-for-fields-that-need-fingerprint-updating-test-5+ (is (= [:and+ [:= :active true]+ [:or+ [:not (app-db/isa :semantic_type :type/PK)]+ [:= :semantic_type nil]]+ [:not-in :visibility_type ["retired" "sensitive"]]+ [:not-in :base_type skip-fingerprint-base-types]+ [:or+ [:and+ [:< :fingerprint_version 4]+ [:in :base_type #{"type/Decimal" "type/Latitude" "type/Longitude" "type/Coordinate" "type/Currency" "type/Float"+ "type/Share" "type/Income" "type/Price" "type/Discount" "type/GrossMargin" "type/Cost" "type/Percentage"}]]+ [:and+ [:< :fingerprint_version 3]+ [:in :base_type #{"type/URL" "type/ImageURL" "type/AvatarURL"}]]+ ;; version 2 can be eliminated completely since everything relevant there is included in 4+ ;; The only things that should go in 1 should be `:type/City` since `:type/Coordinate` is included in 4+ [:and+ [:< :fingerprint_version 1]+ [:in :base_type #{"type/City"}]]]]+ (#'sync.db/needs-fingerprint-update-clause false {1 #{:type/Coordinate :type/City}+ 2 #{:type/Coordinate}+ 3 #{:type/URL}+ 4 #{:type/Float}})))))++(deftest ^:parallel needs-fingerprint-update-clause-test-5(testing "when refingerprinting doesn't check for versions"- (is (= {:where [:and- [:= :active true]- [:or- [:not (app-db/isa :semantic_type :type/PK)]- [:= :semantic_type nil]]- [:not-in :visibility_type ["retired" "sensitive"]]- [:not-in :base_type skip-fingerprint-base-types]]}- (binding [sync.fingerprint/*refingerprint?* true]- (#'sync.fingerprint/honeysql-for-fields-that-need-fingerprint-updating))))))+ (is (= [:and+ [:= :active true]+ [:or+ [:not (app-db/isa :semantic_type :type/PK)]+ [:= :semantic_type nil]]+ [:not-in :visibility_type ["retired" "sensitive"]]+ [:not-in :base_type skip-fingerprint-base-types]]+ (#'sync.db/needs-fingerprint-update-clause true {1 #{:type/URL}})))));; Make sure that the above functions are used correctly to determine which Fields get (re-)fingerprinted(defn- field-was-fingerprinted?! [fingerprint-versions field-properties]