Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Address the remaining db.clj review findings: metabot, sync, search and friends (#81865)

Alexander Polyankina865fd1ad737
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 34fae22741b..88b24b589af 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -407,8 +407,7 @@
auth-identity
{:team "UX West"
- :api #{metabase.auth-identity.core
- metabase.auth-identity.init}
+ :api #{metabase.auth-identity.core metabase.auth-identity.db metabase.auth-identity.init}
:uses #{channel
events
login-history
@@ -1499,7 +1498,8 @@
models
{:team "DevEx"
- :api #{metabase.models.humanization
+ :api #{metabase.models.db
+ metabase.models.humanization
metabase.models.init
metabase.models.interface
metabase.models.resolution
diff --git a/enterprise/backend/src/metabase_enterprise/stale/api.clj b/enterprise/backend/src/metabase_enterprise/stale/api.clj
index 5b751bbda3f..6a9a2d35445 100644
--- a/enterprise/backend/src/metabase_enterprise/stale/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/stale/api.clj
@@ -25,7 +25,7 @@
(comp (map :id)
(remove #(= % (collection/trash-collection-id)))
(filter #(collection/visible-collection-id? % :write)))
- (collection/descendants-flat collection [:= :archived false])))
+ (collection/descendants-flat collection false)))
(defmulti present-model-items
"Given a model and a list of items, return the items in the format the API client expects. Note that order does not
diff --git a/src/metabase/auth_identity/db.clj b/src/metabase/auth_identity/db.clj
index e08a3d53ae8..f5dd76b0770 100644
--- a/src/metabase/auth_identity/db.clj
+++ b/src/metabase/auth_identity/db.clj
@@ -46,7 +46,9 @@
(t2/delete! :model/AuthIdentity :user_id user-id :provider provider))
(defn delete-sessions-for-user!
- "Delete every Session of the User with `user-id`."
+ "Delete every Session of the User with `user-id`. Duplicates `metabase.session.db/delete-sessions-for-user!`;
+ can't delegate to it because the `session` module already depends on `auth-identity`, so the reverse dependency
+ would be a module cycle."
[user-id]
(t2/delete! :model/Session :user_id user-id))
diff --git a/src/metabase/collections/db.clj b/src/metabase/collections/db.clj
index 1aecfce4fe0..031301eef72 100644
--- a/src/metabase/collections/db.clj
+++ b/src/metabase/collections/db.clj
@@ -105,10 +105,24 @@
[collection-ids]
(t2/select [:model/Collection :name :id :personal_owner_id] :id [:in collection-ids] {:order-by [:location]}))
-(defn descendant-summaries-where
- "The name, ID, location, and description of the Collections matching the Honey SQL `where`."
- [where]
- (t2/select [:model/Collection :name :id :location :description] {:where where}))
+(defn descendant-summaries
+ "The name, ID, location, and description of the descendant Collections of the Collection at
+ `children-location-prefix` (see `metabase.collections.models.collection/children-location`), excluding other
+ users' Personal Collections (Personal Collections owned by `current-user-id` are still included). When
+ `archived?` is given (true or false, not nil), further restricted to that archived status.
+ `additional-where-clauses` are ANDed in as-is; used by callers that need a permission-filter builder like
+ `visible-collection-filter-clause`, which lives in `metabase.collections.models.collection` and so can't be
+ called from here without a require cycle (that namespace already requires this one)."
+ [children-location-prefix current-user-id archived? additional-where-clauses]
+ (t2/select [:model/Collection :name :id :location :description]
+ {:where (into [:and
+ [:like :location (str children-location-prefix "%")]
+ [:or
+ [:= :personal_owner_id nil]
+ [:= :personal_owner_id current-user-id]]
+ (when (some? archived?)
+ [:= :archived archived?])]
+ additional-where-clauses)}))
(defn descendant-summaries-with-type
"The name, ID, location, description, and type of the Collections directly under any of `location-prefixes`
diff --git a/src/metabase/collections/models/collection.clj b/src/metabase/collections/models/collection.clj
index eaee77b83a1..587827335ee 100644
--- a/src/metabase/collections/models/collection.clj
+++ b/src/metabase/collections/models/collection.clj
@@ -1117,24 +1117,17 @@
[:ref ::children]])
(mu/defn descendants-flat :- [:sequential CollectionWithLocationAndIDOrRoot]
- "Return all descendant collections of a `collection`, including children, grandchildren, and so forth."
- [collection :- CollectionWithLocationAndIDOrRoot, & additional-honeysql-where-clauses]
- (or
- (collections.db/descendant-summaries-where
- (apply
- vector
- :and
- [:like :location (str (children-location collection) "%")]
- ;; Only return the Personal Collection belonging to the Current
- ;; User, regardless of whether we should actually be allowed to see
- ;; it (e.g., admins have perms for all Collections). This is done
- ;; to keep the Root Collection View for admins from getting crazily
- ;; cluttered with Personal Collections belonging to other users
- [:or
- [:= :personal_owner_id nil]
- [:= :personal_owner_id api/*current-user-id*]]
- additional-honeysql-where-clauses))
- []))
+ "Return all descendant collections of a `collection`, including children, grandchildren, and so forth. Excludes
+ other users' Personal Collections, regardless of whether we should actually be allowed to see them (e.g. admins
+ have perms for all Collections) -- this is done to keep the Root Collection View for admins from getting crazily
+ cluttered with Personal Collections belonging to other users. `archived?`, if given (true or false), restricts
+ to Collections with that archived status. `additional-honeysql-where-clauses` are for permission-filter builders
+ like [[visible-collection-filter-clause]]."
+ ([collection :- CollectionWithLocationAndIDOrRoot]
+ (descendants-flat collection nil))
+ ([collection :- CollectionWithLocationAndIDOrRoot, archived? :- [:maybe :boolean], & additional-honeysql-where-clauses]
+ (collections.db/descendant-summaries
+ (children-location collection) api/*current-user-id* archived? additional-honeysql-where-clauses)))
(mu/defn descendants-flat-for :- [:sequential CollectionWithLocationAndIDOrRoot]
"Like [[descendants-flat]], but returns the descendants of *any* of
@@ -1160,10 +1153,10 @@
where each letter represents a Collection, and the arrows represent values of its respective `:children`
set."
- [collection :- CollectionWithLocationAndIDOrRoot, & additional-honeysql-where-clauses]
+ [collection :- CollectionWithLocationAndIDOrRoot]
;; first, fetch all the descendants of the `collection`, and build a map of location -> children. This will be used
;; so we can fetch the immediate children of each Collection
- (let [location->children (group-by :location (apply descendants-flat collection additional-honeysql-where-clauses))
+ (let [location->children (group-by :location (descendants-flat collection))
;; Next, build a function to add children to a given `coll`. This function will recursively call itself to add
;; children to each child
add-children (fn add-children [coll]
diff --git a/src/metabase/collections_rest/api.clj b/src/metabase/collections_rest/api.clj
index 6851258e24e..776e57298de 100644
--- a/src/metabase/collections_rest/api.clj
+++ b/src/metabase/collections_rest/api.clj
@@ -839,9 +839,10 @@
(defn- annotate-collections
[parent-coll colls {:keys [show-dashboard-questions?]}]
- (let [descendant-collections (collection/descendants-flat parent-coll (collection/visible-collection-filter-clause
- :id
- {:include-archived-items :all}))
+ (let [descendant-collections (collection/descendants-flat parent-coll nil
+ (collection/visible-collection-filter-clause
+ :id
+ {:include-archived-items :all}))
descendant-collection-ids (mapv u/the-id descendant-collections)
diff --git a/src/metabase/metabot/api/metabot.clj b/src/metabase/metabot/api/metabot.clj
index 173815c7b01..bc6ab44aeff 100644
--- a/src/metabase/metabot/api/metabot.clj
+++ b/src/metabase/metabot/api/metabot.clj
@@ -4,11 +4,9 @@
[metabase.api.common :as api]
[metabase.api.macros :as api.macros]
[metabase.api.routes.common :refer [+auth]]
- [metabase.app-db.core :as mdb]
[metabase.metabot.db :as metabot.db]
[metabase.metabot.suggested-prompts :as metabot.suggested-prompts]
[metabase.metabot.task.suggested-prompts-refresh :as metabot.suggested-prompts-refresh]
- [metabase.metabot.tools.util :as metabot.tools.u]
[metabase.metabot.usage :as metabot.usage]
[metabase.premium-features.core :as premium-features]
[metabase.request.core :as request]
@@ -100,30 +98,9 @@
[:sample {:optional true} :boolean]
[:model {:optional true} [:enum "metric" "model"]]
[:model_id {:optional true} pos-int?]]]
- (let [offset (when-not sample (request/offset))
- rand-fn (case (mdb/db-type)
- :postgres :random
- :rand)
- base-query (cond-> {:join [[^:allow-subquery {:select [:id :name :type]
- :from [[(metabot.tools.u/metabot-metrics-and-models-query id)
- :scope]]}
- :card]
- [:and
- [:= :card.id :metabot_prompt.card_id]]]
- :where [:and
- [:= :metabot_prompt.metabot_id id]]}
- model (update :where conj [:= :card.type model])
- model_id (update :where conj [:= :card.id model_id]))
- total (metabot.db/prompt-count-where base-query)
- order-by (if sample
- [[[rand-fn]]]
- [[:card.name :asc]
- [:id :asc]])
- prompts (metabot.db/prompts-where
- (cond-> base-query
- true (assoc :order-by order-by)
- (request/limit) (assoc :limit (request/limit))
- offset (assoc :offset offset)))]
+ (let [offset (when-not sample (request/offset))
+ total (metabot.db/prompt-count id model model_id)
+ prompts (metabot.db/prompts id model model_id sample (request/limit) offset)]
{:prompts prompts
:limit (request/limit)
:offset offset
diff --git a/src/metabase/metabot/db.clj b/src/metabase/metabot/db.clj
index 2b441d126d8..1c1d0d4c040 100644
--- a/src/metabase/metabot/db.clj
+++ b/src/metabase/metabot/db.clj
@@ -3,8 +3,18 @@
additional logic, so the rest of the module only touches `toucan2.core` for model definitions, hydration methods,
and transactions."
(:require
+ [clojure.string :as str]
+ [metabase.api.common :as api]
+ [metabase.app-db.core :as mdb]
+ [metabase.audit-app.core :as audit-app]
+ [metabase.collections.models.collection :as collection.model]
+ [metabase.models.interface :as mi]
+ [metabase.premium-features.core :as premium-features]
+ [metabase.util :as u]
[toucan2.core :as t2]))
+(declare collection metabot-metrics-and-models-query root-collections-of-types)
+
;;; ------------------------------------------------- Metabot -------------------------------------------------
(defn metabot
@@ -44,9 +54,32 @@
[metabot-ids]
(t2/select :model/MetabotPrompt {:where [:in :metabot_id metabot-ids]}))
-(defn prompts-where
- "The prompt, model, and Card columns of the MetabotPrompts matching the Honey SQL `query`."
- [query]
+(defn- prompts-for-metabot-query
+ "Honey SQL `:join`/`:where` restricting to MetabotPrompts of the Metabot with `metabot-id` whose Card is within
+ scope, optionally further restricted to Cards of `card-type` or the Card with `card-id`."
+ [metabot-id card-type card-id]
+ (cond-> {:join [[^:allow-subquery {:select [:id :name :type]
+ :from [[(metabot-metrics-and-models-query metabot-id) :scope]]}
+ :card]
+ [:and
+ [:= :card.id :metabot_prompt.card_id]]]
+ :where [:and
+ [:= :metabot_prompt.metabot_id metabot-id]]}
+ card-type (update :where conj [:= :card.type card-type])
+ card-id (update :where conj [:= :card.id card-id])))
+
+(defn- prompt-sample-order-by
+ "A random `:order-by` clause, using the database's native random function."
+ []
+ [[[(case (mdb/db-type)
+ :postgres :random
+ :rand)]]])
+
+(defn prompts
+ "The prompt, model, and Card columns of the MetabotPrompts of the Metabot with `metabot-id` whose Card is within
+ scope, optionally restricted to Cards of `card-type` or the Card with `card-id`, ordered randomly if `sample?` else
+ by Card name, and limited/offset by `limit`/`offset`."
+ [metabot-id card-type card-id sample? limit offset]
(t2/select [:model/MetabotPrompt
:id
:prompt
@@ -55,12 +88,18 @@
[:card.name :model_name]
:created_at
:updated_at]
- query))
-
-(defn prompt-count-where
- "The number of MetabotPrompts matching the Honey SQL `query`."
- [query]
- (t2/count :model/MetabotPrompt query))
+ (cond-> (prompts-for-metabot-query metabot-id card-type card-id)
+ true (assoc :order-by (if sample?
+ (prompt-sample-order-by)
+ [[:card.name :asc] [:id :asc]]))
+ limit (assoc :limit limit)
+ offset (assoc :offset offset))))
+
+(defn prompt-count
+ "The number of MetabotPrompts of the Metabot with `metabot-id` whose Card is within scope, optionally restricted to
+ Cards of `card-type` or the Card with `card-id`."
+ [metabot-id card-type card-id]
+ (t2/count :model/MetabotPrompt (prompts-for-metabot-query metabot-id card-type card-id)))
(defn prompt-count-for-metabot
"The number of MetabotPrompts of the Metabot with `metabot-id`."
@@ -358,36 +397,59 @@
:active true
:visibility_type nil))
-(defn visible-table-summaries-where
+(defn- current-user-visible-table-clause
+ "Honey SQL `{:where …}` (plus `:with` when the filter needs a CTE) restricting Tables to those visible to the
+ current user for querying."
+ []
+ (let [{table-where-clause :clause table-cte :with}
+ (mi/visible-filter-clause :model/Table
+ :id
+ {:user-id api/*current-user-id*
+ :is-superuser? api/*is-superuser?*}
+ {:perms/view-data :unrestricted
+ :perms/create-queries :query-builder-and-native})]
+ (cond-> {:where table-where-clause}
+ table-cte (assoc :with table-cte))))
+
+(defn visible-table-summaries-for-current-user
"The ID, name, schema, and description of the active, unhidden Tables among `table-ids` in the Database with
- `database-id` matching the Honey SQL `query`."
- [database-id table-ids query]
+ `database-id` that are visible to the current user for querying."
+ [database-id table-ids]
(t2/select [:model/Table :id :name :schema :description]
:db_id database-id
:id [:in table-ids]
:active true
:visibility_type nil
- query))
+ (current-user-visible-table-clause)))
+
+(def ^:private max-visible-tables-to-consider
+ "Cap on the number of visible Tables fetched for fuzzy table-name matching."
+ 10000)
-(defn visible-tables-reducible
- "Reducible ID, name, schema, and description of the active, unhidden Tables in the Database with `database-id`
- matching the Honey SQL `query`."
- [database-id query]
+(defn visible-tables-excluding
+ "Reducible ID, name, schema, and description of up to [[max-visible-tables-to-consider]] active, unhidden Tables in
+ the Database with `database-id` that are visible to the current user for querying, excluding `excluded-table-ids`."
+ [database-id excluded-table-ids]
(t2/reducible-select [:model/Table :id :name :schema :description]
:db_id database-id
:active true
:visibility_type nil
- query))
-
-(defn most-viewed-tables-where
- "The ID, Database ID, name, schema, and description of the active, unhidden Tables in the Database with
- `database-id` matching the Honey SQL `query`."
- [database-id query]
+ (cond-> (assoc (current-user-visible-table-clause) :limit max-visible-tables-to-consider)
+ (seq excluded-table-ids)
+ (update :where (fn [where-clause]
+ (if where-clause
+ [:and where-clause [:not-in :id excluded-table-ids]]
+ [:not-in :id excluded-table-ids]))))))
+
+(defn most-viewed-tables-visible-to-current-user
+ "The ID, Database ID, name, schema, and description of up to `limit` active, unhidden Tables in the Database with
+ `database-id` that are visible to the current user for querying, most viewed first."
+ [database-id limit]
(t2/select [:model/Table :id :db_id :name :schema :description]
:db_id database-id
:active true
:visibility_type nil
- query))
+ (assoc (current-user-visible-table-clause) :order-by [[:view_count :desc]] :limit limit)))
(defn table-names
"Up to `limit` IDs, names, and schemas of the active, unhidden Tables in the Database with `database-id`."
@@ -423,21 +485,60 @@
:where [:and [:= :db_id database-id] [:= :active true]]
:order-by [[:schema :asc]]}))
-(defn query-table-reference-where
- "The first Table ID, name, and schema matching the Honey SQL `where`, as a query table reference."
- [where]
+(def ^:private quoted-identifier-chars
+ "Characters that quote a SQL identifier, making its match against a Table name/schema case-sensitive."
+ "\"`")
+
+(defn- quote-stripper
+ [quote-char]
+ (let [doubled (str quote-char quote-char)
+ single (str quote-char)]
+ #(-> (subs % 1 (dec (count %)))
+ (str/replace doubled single))))
+
+(def ^:private quote-char->stripper
+ (zipmap quoted-identifier-chars (map quote-stripper quoted-identifier-chars)))
+
+(defn- table-part-clause
+ "Exact match for a quoted `value`, case-insensitive match for an unquoted `value`. Case-insensitive matching is not
+ correct for every database (Oracle/Postgres/H2 all treat unquoted identifiers differently), but MySQL is truly
+ case-insensitive, so this caters to the lowest common denominator; identifiers differing only by case are already
+ an anti-pattern, so this leniency is unlikely to cause issues in practice."
+ [field value]
+ (if-let [strip (quote-char->stripper (first value))]
+ [:= field (strip value)]
+ [:= [:lower field] (u/lower-case-en value)]))
+
+(defn- table-match-clause
+ [{:keys [schema table]}]
+ (if-not schema
+ (table-part-clause :t.name table)
+ [:and
+ (table-part-clause :t.name table)
+ (table-part-clause :t.schema schema)]))
+
+(defn query-table-reference
+ "The first Table ID, name, and schema in the Database with `db-id` matching `table` (and `schema`, if given), as a
+ query table reference. Matching is case-insensitive unless `table`/`schema` are quoted with `\"` or `` ` ``."
+ [db-id schema table]
(t2/select-one :model/QueryTable
{:select [[:t.id :table-id] [:t.name :table] [:t.schema :schema]]
:from [[(t2/table-name :model/Table) :t]]
- :where where}))
-
-(defn query-table-references-where
- "The Table IDs, names, and schemas matching the Honey SQL `where`, as query table references."
- [where]
+ :where [:and
+ [:= :t.db_id db-id]
+ (table-match-clause {:schema schema :table table})]}))
+
+(defn query-table-references
+ "The Table IDs, names, and schemas in the Database with `db-id` matching any of `tables` (each a map of `:schema`
+ and `:table`), as query table references. Matching is case-insensitive unless quoted, as in
+ [[query-table-reference]]."
+ [db-id tables]
(t2/select :model/QueryTable
{:select [[:t.id :table-id] [:t.name :table] [:t.schema :schema]]
:from [[(t2/table-name :model/Table) :t]]
- :where where}))
+ :where [:and
+ [:= :t.db_id db-id]
+ (into [:or] (map table-match-clause) tables)]}))
;;; ------------------------------------------------- Fields -------------------------------------------------
@@ -478,6 +579,71 @@
[card-id]
(t2/select-one [:model/Card :id :type :card_schema] :id card-id))
+(defn metabot-metrics-and-models-query
+ "Honey SQL query selecting the metric and model Cards in scope of the Metabot with `metabot-id` that are visible to
+ the current user, ignoring analytics content. If the Metabot has `:use_verified_content` enabled, restricts to
+ verified-or-curated content (verified, official-collection, or library-published). `limit`, if given, caps the
+ number of rows."
+ [metabot-id & {:keys [limit]}]
+ (let [metabot-instance (metabot metabot-id)
+ metabot-collection-id (:collection_id metabot-instance)
+ use-verified-content? (:use_verified_content metabot-instance)
+ verified? (premium-features/has-feature? :content-verification)
+ official? (premium-features/has-feature? :official-collections)
+ library? (premium-features/has-feature? :library)
+ ;; ids of collections under a Library-type root; their metrics/models are library-published content
+ library-coll-ids (when library?
+ (let [roots (root-collections-of-types (mapv name collection.model/library-collection-types))]
+ (into (set (map :id roots)) (mapcat collection.model/descendant-ids roots))))
+ ;; Mirror collections.curation/curated? for card scope: verified, official-collection, or
+ ;; library-published (under a Library root). Each disjunct is gated on its feature.
+ curated-conds (cond-> []
+ verified? (conj [:= :mr.status "verified"])
+ official? (conj [:= :collection.authority_level "official"])
+ (seq library-coll-ids) (conj [:in :report_card.collection_id (vec library-coll-ids)]))
+ ;; Columns are qualified with report_card because the official-collections branch joins
+ ;; `collection`, which shares column names (type, archived, id) — unqualified refs would be ambiguous.
+ collection-filter (if metabot-collection-id
+ (let [metabot-collection (collection metabot-collection-id)
+ collection-ids (conj (collection.model/descendant-ids metabot-collection) metabot-collection-id)]
+ [:in :report_card.collection_id collection-ids])
+ [:and true])
+ base-query ^:allow-subquery {:select [:report_card.*]
+ :from [[:report_card]]
+ :where [:and
+ [:!= :report_card.database_id audit-app/audit-db-id]
+ collection-filter
+ [:in :report_card.type ["metric" "model"]]
+ [:= :report_card.archived false]
+ (when api/*current-user-id*
+ (collection.model/visible-collection-filter-clause :report_card.collection_id))]}]
+ (cond-> base-query
+ verified?
+ (update :left-join (fnil into []) [[:moderation_review :mr] [:and
+ [:= :mr.moderated_item_id :report_card.id]
+ [:= :mr.moderated_item_type "card"]
+ [:= :mr.most_recent true]]])
+
+ official?
+ (update :left-join (fnil into []) [[:collection :collection]
+ [:= :collection.id :report_card.collection_id]])
+
+ ;; Prioritize curated content.
+ (seq curated-conds)
+ (assoc :order-by [[[:case (into [:or] curated-conds) [:inline 0] :else [:inline 1]] :asc]])
+
+ ;; Restrict to curated content only when that's desired.
+ (and use-verified-content? (seq curated-conds))
+ (update :where conj (into [:or] curated-conds))
+
+ ;; Setting on but no curation features active → nothing is curated, so return nothing rather than
+ ;; falling through unfiltered to uncurated cards.
+ (and use-verified-content? (empty? curated-conds))
+ (update :where conj [:= [:inline 1] [:inline 0]])
+
+ (integer? limit)
+ (assoc :limit limit))))
+
(defn cards-where
"The Cards matching the Honey SQL `query`."
[query]
diff --git a/src/metabase/metabot/query_analyzer.clj b/src/metabase/metabot/query_analyzer.clj
index 8cca3914868..3146a76ff22 100644
--- a/src/metabase/metabot/query_analyzer.clj
+++ b/src/metabase/metabot/query_analyzer.clj
@@ -36,41 +36,12 @@
(defn- normalized-key [value]
(u/lower-case-en (strip-quotes value)))
-(defn- field-query
- "Exact match for quoted fields, case-insensitive match for non-quoted fields"
- [field value]
- (if-let [f (quote->stripper (first value))]
- [:= field (f value)]
- ;; Technically speaking, this is not correct for all databases.
- ;;
- ;; For example, Oracle treats non-quoted identifiers as uppercase, but still expects a case-sensitive match.
- ;; Similarly, Postgres treats all non-quoted identifiers as lowercase, and again expects an exact match.
- ;; H2 on the other hand will choose whether to cast it to uppercase or lowercase based on a system variable... T_T
- ;;
- ;; MySQL, by contrast, is truly case-insensitive, and as the lowest common denominator it's what we cater for.
- ;; In general, it's a huge anti-pattern to have any identifiers that differ only by case, so this extra leniency is
- ;; unlikely to ever cause issues in practice.
- ;;
- ;; If we want 100% correctness, we can use the Macaw :case-insensitive option here to do the right thing.
- [:= [:lower field] (u/lower-case-en value)]))
-
-(defn- table-query
- [{:keys [schema table]}]
- (if-not schema
- (field-query :t.name table)
- [:and
- (field-query :t.name table)
- (field-query :t.schema schema)]))
-
(defn table-reference
"Used by tests"
([db-id table]
(table-reference db-id nil table))
([db-id schema table]
- (metabot.db/query-table-reference-where [:and
- [:= :t.db_id db-id]
- (table-query {:schema (some-> schema name)
- :table (name table)})])))
+ (metabot.db/query-table-reference db-id (some-> schema name) (name table))))
(defn- strip-redundant-table-refs
"Strip out duplicate references, and unqualified references that are shadowed by found or qualified ones."
@@ -100,9 +71,7 @@
(consolidate-tables
tables
(when (seq tables)
- (metabot.db/query-table-references-where [:and
- [:= :t.db_id db-id]
- (into [:or] (map table-query tables))]))))
+ (metabot.db/query-table-references db-id tables))))
(defn- tables-via-sql-tools
"Returns a set of table identifiers that (may) be referenced in the given card's query.
diff --git a/src/metabase/metabot/table_utils.clj b/src/metabase/metabot/table_utils.clj
index 2ec46c415a9..a47a52fc49a 100644
--- a/src/metabase/metabot/table_utils.clj
+++ b/src/metabase/metabot/table_utils.clj
@@ -2,10 +2,8 @@
"Shared table utilities for enterprise modules."
(:require
[clojure.set :as set]
- [metabase.api.common :as api]
[metabase.metabot.db :as metabot.db]
[metabase.metabot.query-analyzer :as query-analyzer]
- [metabase.models.interface :as mi]
[metabase.util :as u]
[toucan2.core :as t2]
[toucan2.realize :as t2.realize])
@@ -40,18 +38,8 @@
priority-tables []
exclude-table-ids #{}}}]
(let [priority-table-ids (set (map :id priority-tables))
- {table-where-clause :clause table-cte :with} (mi/visible-filter-clause :model/Table
- :id
- {:user-id api/*current-user-id*
- :is-superuser? api/*is-superuser?*}
- {:perms/view-data :unrestricted
- :perms/create-queries :query-builder-and-native})
;; Fetch most viewed tables, excluding priority tables and excluded tables
- fill-tables (metabot.db/most-viewed-tables-where database-id
- (cond-> {:where table-where-clause
- :order-by [[:view_count :desc]]
- :limit all-tables-limit}
- table-cte (assoc :with table-cte)))
+ fill-tables (metabot.db/most-viewed-tables-visible-to-current-user database-id all-tables-limit)
fill-tables (remove #(or (priority-table-ids (:id %))
(exclude-table-ids (:id %))) fill-tables)
fill-tables (t2/hydrate fill-tables :fields)
@@ -108,17 +96,6 @@
(nil? tschema)
(similar? uschema tschema))))
-(defn- visible-filter-clause
- []
- (let [{table-where-clause :clause table-cte :with} (mi/visible-filter-clause :model/Table
- :id
- {:user-id api/*current-user-id*
- :is-superuser? api/*is-superuser?*}
- {:perms/view-data :unrestricted
- :perms/create-queries :query-builder-and-native})]
- (cond-> {:where table-where-clause}
- table-cte (assoc :with table-cte))))
-
(defn find-matching-tables
"Find tables in the database that are similar to the unrecognized tables using fuzzy matching.
@@ -139,12 +116,7 @@
(keep (fn [table]
(when (some #(matching-tables? table % {:match-schema? false}) unrecognized-tables)
(t2.realize/realize table))))
- (metabot.db/visible-tables-reducible database-id
- (cond-> (assoc (visible-filter-clause)
- :limit 10000)
- (seq used-ids) (update :where #(if %
- [:and % [:not-in :id used-ids]]
- [:not-in :id used-ids]))))))
+ (metabot.db/visible-tables-excluding database-id used-ids)))
(defn used-tables-from-ids
"Return table info for `table-ids` in the same shape as [[used-tables]].
@@ -153,7 +125,7 @@
[database-id table-ids]
(if-not (seq table-ids)
[]
- (metabot.db/visible-table-summaries-where database-id table-ids (visible-filter-clause))))
+ (metabot.db/visible-table-summaries-for-current-user database-id table-ids)))
(defn used-tables
"Return all tables used in the query, including fuzzy-matched ones.
diff --git a/src/metabase/metabot/tools/util.clj b/src/metabase/metabot/tools/util.clj
index 99419fbfc68..12b60bb8f00 100644
--- a/src/metabase/metabot/tools/util.clj
+++ b/src/metabase/metabot/tools/util.clj
@@ -2,14 +2,11 @@
(:require
[medley.core :as m]
[metabase.api.common :as api]
- [metabase.audit-app.core :as audit-app]
- [metabase.collections.models.collection :as collection]
[metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.lib.types.isa :as lib.types.isa]
[metabase.metabot.db :as metabot.db]
- [metabase.premium-features.core :as premium-features]
[metabase.util :as u]))
(defn handle-agent-error
@@ -193,74 +190,6 @@
(let [mp (lib-be/application-database-metadata-provider (:db_id table))]
(lib/query mp (lib.metadata/table mp table-id)))))
-(defn metabot-metrics-and-models-query
- "Return the metric and model cards in metabot scope visible to the current user.
-
- Takes a metabot-id and returns all metric and model cards in that metabot's collection
- and its subcollections. If the metabot has use_verified_content enabled, only verified-or-curated
- content is returned — verified, official-collection, or library-published cards.
-
- Ignores analytics content."
- [metabot-id & {:keys [limit] :as _opts}]
- (let [metabot (metabot.db/metabot metabot-id)
- metabot-collection-id (:collection_id metabot)
- use-verified-content? (:use_verified_content metabot)
- verified? (premium-features/has-feature? :content-verification)
- official? (premium-features/has-feature? :official-collections)
- library? (premium-features/has-feature? :library)
- ;; ids of collections under a Library-type root; their metrics/models are library-published content
- library-coll-ids (when library?
- (let [roots (metabot.db/root-collections-of-types (mapv name collection/library-collection-types))]
- (into (set (map :id roots)) (mapcat collection/descendant-ids roots))))
- ;; Mirror collections.curation/curated? for card scope: verified, official-collection, or
- ;; library-published (under a Library root). Each disjunct is gated on its feature.
- curated-conds (cond-> []
- verified? (conj [:= :mr.status "verified"])
- official? (conj [:= :collection.authority_level "official"])
- (seq library-coll-ids) (conj [:in :report_card.collection_id (vec library-coll-ids)]))
- ;; Columns are qualified with report_card because the official-collections branch joins
- ;; `collection`, which shares column names (type, archived, id) — unqualified refs would be ambiguous.
- collection-filter (if metabot-collection-id
- (let [collection (metabot.db/collection metabot-collection-id)
- collection-ids (conj (collection/descendant-ids collection) metabot-collection-id)]
- [:in :report_card.collection_id collection-ids])
- [:and true])
- base-query ^:allow-subquery {:select [:report_card.*]
- :from [[:report_card]]
- :where [:and
- [:!= :report_card.database_id audit-app/audit-db-id]
- collection-filter
- [:in :report_card.type ["metric" "model"]]
- [:= :report_card.archived false]
- (when api/*current-user-id*
- (collection/visible-collection-filter-clause :report_card.collection_id))]}]
- (cond-> base-query
- verified?
- (update :left-join (fnil into []) [[:moderation_review :mr] [:and
- [:= :mr.moderated_item_id :report_card.id]
- [:= :mr.moderated_item_type "card"]
- [:= :mr.most_recent true]]])
-
- official?
- (update :left-join (fnil into []) [[:collection :collection]
- [:= :collection.id :report_card.collection_id]])
-
- ;; Prioritize curated content.
- (seq curated-conds)
- (assoc :order-by [[[:case (into [:or] curated-conds) [:inline 0] :else [:inline 1]] :asc]])
-
- ;; Restrict to curated content only when that's desired.
- (and use-verified-content? (seq curated-conds))
- (update :where conj (into [:or] curated-conds))
-
- ;; Setting on but no curation features active → nothing is curated, so return nothing rather than
- ;; falling through unfiltered to uncurated cards.
- (and use-verified-content? (empty? curated-conds))
- (update :where conj [:= [:inline 1] [:inline 0]])
-
- (integer? limit)
- (assoc :limit limit))))
-
(defn- destination-db-ids
"Returns the subset of `db-ids` that back a destination (routed) database -- routing internals
reachable only through their router database (see
@@ -275,7 +204,7 @@
Only cards visible to the current user are returned, excluding those backed by a destination
(routed) database (see [[destination-db-ids]])."
[metabot-id & {:as opts}]
- (let [cards (metabot.db/cards-where (-> (metabot-metrics-and-models-query metabot-id opts)
+ (let [cards (metabot.db/cards-where (-> (metabot.db/metabot-metrics-and-models-query metabot-id opts)
;; qualified: the official-collections branch joins `collection`,
;; which also has `id`
(update :order-by (fnil conj []) [:report_card.id])))
diff --git a/src/metabase/models/db.clj b/src/metabase/models/db.clj
index a7f2ff298ce..7c5f45ef2aa 100644
--- a/src/metabase/models/db.clj
+++ b/src/metabase/models/db.clj
@@ -203,6 +203,13 @@
(when (seq field-names)
(t2/select-one-pk :model/Field (field-in-path-query table-id field-names))))
+(defn field-in-path
+ "The Field named by the last of `field-names` (each nested inside the previous, bottom-most first) under
+ `table-id`, or nil."
+ [table-id field-names]
+ (when (seq field-names)
+ (t2/select-one :model/Field (field-in-path-query table-id field-names))))
+
(defn insert-inactive-field!
"Insert an inactive, untyped Field named `field-name` under `parent-id` in the Table with `table-id` and return its
id."
diff --git a/src/metabase/search/appdb/core.clj b/src/metabase/search/appdb/core.clj
index 7a3301e6ffc..15ed88f1a6c 100644
--- a/src/metabase/search/appdb/core.clj
+++ b/src/metabase/search/appdb/core.clj
@@ -173,7 +173,7 @@
scorers (search.scoring/scorers search-ctx)
query (->> (base-filtered-query search-ctx search-string [:legacy_input])
(search.scoring/with-scores search-ctx scorers))]
- (->> (search.db/rows query)
+ (->> (search.db/scored-search-rows query)
(map (partial rehydrate weights (keys scorers)))))
(catch Exception e
;; Rule out the error coming from stale index metadata.
@@ -197,7 +197,7 @@
:search_index.model
:search_index.source_type)))
(search.filter/with-filters search-ctx)
- search.db/rows
+ search.db/distinct-model-rows
(into #{} (map :model)))))
(defn- restrict-to-row [model id qry]
@@ -209,7 +209,7 @@
(-> qry
(assoc :select [[[:inline 1] :one]] :limit 1)
(dissoc :order-by)
- search.db/rows
+ search.db/search-index-probe-rows
seq
boolean))
diff --git a/src/metabase/search/appdb/index.clj b/src/metabase/search/appdb/index.clj
index b6b08abc551..3dde711bded 100644
--- a/src/metabase/search/appdb/index.clj
+++ b/src/metabase/search/appdb/index.clj
@@ -2,7 +2,6 @@
(:require
[clojure.set :as set]
[clojure.string :as str]
- [honey.sql.helpers :as sql.helpers]
[metabase.analytics-interface.core :as analytics]
[metabase.app-db.core :as mdb]
[metabase.config.core :as config]
@@ -114,7 +113,7 @@
(defn- drop-table! [table]
(boolean
(when table
- (search.db/execute! (sql.helpers/drop-table :if-exists (keyword (table-name table)))))))
+ (search.db/drop-search-index-table-if-exists! (keyword (table-name table))))))
(defn- orphan-indexes []
(map (comp keyword u/lower-case-en :table_name)
@@ -127,7 +126,7 @@
(let [dropped (volatile! [])]
(doseq [table (orphan-indexes)]
(try
- (search.db/execute! (sql.helpers/drop-table table))
+ (search.db/drop-search-index-table! table)
(vswap! dropped conj table)
;; Deletion could fail if it races with other instances
(catch Exception e
@@ -178,12 +177,10 @@
;; Create with a separate transaction so that postgresql will complete the index creations before returning,
;; even when already running in a transaction
(t2/with-transaction [_ (mdb/app-db)]
- (-> (sql.helpers/create-table table-name)
- (sql.helpers/with-columns (specialization/table-schema base-schema))
- search.db/execute!)
+ (search.db/create-search-index-table! table-name (specialization/table-schema base-schema))
(let [table-name (name table-name)]
(doseq [stmt (specialization/post-create-statements table-name table-name)]
- (search.db/execute! stmt)))))
+ (search.db/run-search-index-statement! stmt)))))
(defn maybe-create-pending!
"Create a search index table if one doesn't exist. Record and return the name of the table, regardless."
@@ -432,7 +429,7 @@
"Use the index table to search for records."
[search-term & [search-ctx]]
(map (juxt :model :name)
- (search.db/rows (search-query search-term search-ctx [:model :name]))))
+ (search.db/search-index-rows (search-query search-term search-ctx [:model :name]))))
(defn reset-index!
"Ensure we have a blank slate; in case the table schema or stored data format has changed."
diff --git a/src/metabase/search/appdb/scoring.clj b/src/metabase/search/appdb/scoring.clj
index cf64abaf7b1..d648bd99fad 100644
--- a/src/metabase/search/appdb/scoring.clj
+++ b/src/metabase/search/appdb/scoring.clj
@@ -18,9 +18,7 @@
(defn- view-count-percentiles*
[p-value]
- (into {} (for [{:keys [model vcp]} (search.db/rows (specialization/view-count-percentile-query
- (search.index/active-table)
- p-value))]
+ (into {} (for [{:keys [model vcp]} (search.db/view-count-percentile-rows (search.index/active-table) p-value)]
[(keyword model) vcp])))
(def ^{:private true
diff --git a/src/metabase/search/appdb/specialization/postgres.clj b/src/metabase/search/appdb/specialization/postgres.clj
index 71abfdc83ac..ed438b93d28 100644
--- a/src/metabase/search/appdb/specialization/postgres.clj
+++ b/src/metabase/search/appdb/specialization/postgres.clj
@@ -25,18 +25,7 @@
"CREATE INDEX IF NOT EXISTS %s_archived_idx ON %s (archived)"]))
(defmethod specialization/batch-upsert! :postgres [table entries]
- (when (seq entries)
- (search.db/execute!
- ;; The cost of dynamically calculating these keys should be small compared to the IO cost, so unoptimized. Note
- ;; that the entries are not guaranteed to be homogeneous - some may be missing nullable columns. So we need to
- ;; get the keys from *all* the entries.
- (let [update-keys (vec (disj (set (mapcat keys entries)) :id :model :model_id))
- excluded-kw (fn [column] (keyword (str "excluded." (name column))))]
- {:insert-into table
- :values entries
- :on-conflict [:model :model_id]
- :do-update-set (with-meta (zipmap update-keys (map excluded-kw update-keys))
- {:allow-subquery true})}))))
+ (search.db/postgres-batch-upsert! table entries))
(defmethod specialization/base-query :postgres
[active-table search-term search-ctx select-items]
@@ -103,7 +92,7 @@
(defmethod specialization/analyze-table! :postgres
[table-name]
- (search.db/execute! (str "ANALYZE " (name table-name))))
+ (search.db/analyze-search-index-table! table-name))
(defmethod specialization/index-size-estimate :postgres
[table-name]
diff --git a/src/metabase/search/db.clj b/src/metabase/search/db.clj
index cb0c9ae6289..c6bbc16f97e 100644
--- a/src/metabase/search/db.clj
+++ b/src/metabase/search/db.clj
@@ -2,18 +2,82 @@
"Application database queries for the search module. Every function here is a direct Toucan 2 call with no
additional logic, so no other namespace in the module runs a query itself (connection and transaction handling still use `toucan2.core`)."
(:require
+ [honey.sql.helpers :as sql.helpers]
+ [metabase.search.appdb.specialization.api :as specialization]
[toucan2.core :as t2]))
-(defn rows
- "The rows returned by the Honey SQL or raw SQL `query`."
+(defn spec-index-rows
+ "The rows matching the Honey SQL `query` built from a search model's spec by `metabase.search.ingestion`."
[query]
(t2/query query))
-(defn execute!
- "Run the Honey SQL or raw SQL `statement`."
+(defn scored-search-rows
+ "The rows matching the scored, filtered search Honey SQL `query` built by `metabase.search.appdb.core`."
+ [query]
+ (t2/query query))
+
+(defn distinct-model-rows
+ "The rows matching the Honey SQL `query` built by `metabase.search.appdb.core` to find the distinct search models
+ present in the results."
+ [query]
+ (t2/query query))
+
+(defn search-index-probe-rows
+ "The rows matching the Honey SQL `query` built by `metabase.search.appdb.core` to check whether a single row
+ survives a filter."
+ [query]
+ (t2/query query))
+
+(defn search-index-rows
+ "The rows matching the Honey SQL `query` built by `metabase.search.appdb.index/search-query`."
+ [query]
+ (t2/query query))
+
+(defn view-count-percentile-rows
+ "The Model to view-count-percentile rows for the search index table `index-table` at percentile `p-value`."
+ [index-table p-value]
+ (t2/query (specialization/view-count-percentile-query index-table p-value)))
+
+(defn drop-search-index-table-if-exists!
+ "Drop the search index table named `table-name`, if it exists."
+ [table-name]
+ (t2/query (sql.helpers/drop-table :if-exists table-name)))
+
+(defn drop-search-index-table!
+ "Drop the search index table named `table-name`."
+ [table-name]
+ (t2/query (sql.helpers/drop-table table-name)))
+
+(defn create-search-index-table!
+ "Create the search index table named `table-name` with `columns` (the Honey SQL column definitions built by the
+ active search engine specialization)."
+ [table-name columns]
+ (t2/query (-> (sql.helpers/create-table table-name)
+ (sql.helpers/with-columns columns))))
+
+(defn run-search-index-statement!
+ "Run a single post-creation SQL statement (e.g. an index creation) for a search index table."
[statement]
(t2/query statement))
+(defn analyze-search-index-table!
+ "Run `ANALYZE` on the search index table `table-name` (Postgres only)."
+ [table-name]
+ (t2/query (str "ANALYZE " (name table-name))))
+
+(defn postgres-batch-upsert!
+ "Upsert `entries` into the search index `table`, on conflict of `(model, model_id)` overwriting every other column
+ with the new value."
+ [table entries]
+ (when (seq entries)
+ (let [update-keys (vec (disj (set (mapcat keys entries)) :id :model :model_id))
+ excluded-kw (fn [column] (keyword (str "excluded." (name column))))]
+ (t2/query {:insert-into table
+ :values entries
+ :on-conflict [:model :model_id]
+ :do-update-set (with-meta (zipmap update-keys (map excluded-kw update-keys))
+ {:allow-subquery true})}))))
+
(defn commit!
"Commit the current transaction."
[]
diff --git a/src/metabase/search/ingestion.clj b/src/metabase/search/ingestion.clj
index 308e7e42f44..8d5c5d94c95 100644
--- a/src/metabase/search/ingestion.clj
+++ b/src/metabase/search/ingestion.clj
@@ -224,7 +224,7 @@
(let [spec (search.spec/spec search-model)
indexed? (-> (spec-index-query-where search-model [:= :this.id id])
(assoc :select [[[:inline 1] :one]] :limit 1)
- search.db/rows
+ search.db/spec-index-rows
seq
boolean)]
(cond
@@ -276,7 +276,7 @@
(->> (for [model search.spec/search-models]
(-> (spec-index-query-where model nil)
(assoc :select [[:%count.* :count]])
- search.db/rows
+ search.db/spec-index-rows
first
:count))
(filter some?)
diff --git a/src/metabase/session/db.clj b/src/metabase/session/db.clj
index 3d0abb3df0b..ae00191c010 100644
--- a/src/metabase/session/db.clj
+++ b/src/metabase/session/db.clj
@@ -3,6 +3,7 @@
additional logic, so no other namespace in the module runs a query itself (model definitions still use `toucan2.core`)."
(:require
[metabase.app-db.core :as mdb]
+ [metabase.auth-identity.db :as auth-identity.db]
[metabase.tracing.core :as tracing]
[metabase.util :as u]
[metabase.util.honey-sql-2 :as h2x]
@@ -37,14 +38,15 @@
(t2/query-one hsql))))
(defn auth-identity-for-provider
- "The AuthIdentity of the User with `user-id` at `provider`, or nil."
+ "The AuthIdentity of the User with `user-id` at `provider`, or nil. See `metabase.auth-identity.db/auth-identity`,
+ which owns the AuthIdentity table."
[user-id provider]
- (t2/select-one :model/AuthIdentity :user_id user-id :provider provider))
+ (auth-identity.db/auth-identity user-id provider))
(defn auth-identity-exists?
"Whether the User with `user-id` has an AuthIdentity at `provider`."
[user-id provider]
- (t2/exists? :model/AuthIdentity :user_id user-id :provider provider))
+ (auth-identity.db/auth-identity-exists? user-id provider))
(defn set-auth-identity-credentials!
"Set the `credentials` of the AuthIdentity with `auth-identity-id`."
diff --git a/src/metabase/sync/analyze/fingerprint.clj b/src/metabase/sync/analyze/fingerprint.clj
index c15d67b893c..27dc192d267 100644
--- a/src/metabase/sync/analyze/fingerprint.clj
+++ b/src/metabase/sync/analyze/fingerprint.clj
@@ -2,10 +2,7 @@
"Analysis sub-step that takes a sample of values for a Field and saving a non-identifying fingerprint
used for classification. This fingerprint is saved as a column on the Field it belongs to."
(:require
- [clojure.set :as set]
- [honey.sql.helpers :as sql.helpers]
[metabase.analyze.core :as analyze]
- [metabase.app-db.core :as app-db]
[metabase.driver :as driver]
[metabase.driver.util :as driver.u]
[metabase.lib.schema.metadata.fingerprint :as lib.schema.metadata.fingerprint]
@@ -101,98 +98,11 @@
;;; | WHICH FIELDS NEED UPDATED FINGERPRINTS? |
;;; +----------------------------------------------------------------------------------------------------------------+
-;; Logic for building the somewhat-complicated query we use to determine which Fields need new Fingerprints
-;;
-;; This ends up giving us a SQL query that looks something like:
-;;
-;; SELECT *
-;; FROM metabase_field
-;; WHERE active = true
-;; AND (semantic_type NOT IN ('type/PK') OR semantic_type IS NULL)
-;; AND preview_display = true
-;; AND visibility_type <> 'retired'
-;; AND table_id = 1
-;; AND ((fingerprint_version < 1 AND
-;; base_type IN ("type/Longitude", "type/Latitude", "type/Integer"))
-;; OR
-;; (fingerprint_version < 2 AND
-;; base_type IN ("type/Text", "type/SerializedJSON")))
-
-(mu/defn- base-types->descendants :- [:maybe [:set ms/FieldTypeKeywordOrString]]
- "Given a set of `base-types` return an expanded set that includes those base types as well as all of their
- descendants. These types are converted to strings so HoneySQL doesn't confuse them for columns."
- [base-types :- [:set ms/FieldType]]
- (into #{}
- (comp (mapcat (fn [base-type]
- (cons base-type (descendants base-type))))
- (map u/qualified-name))
- base-types))
-
-;; It's even cooler if we could generate efficient SQL that looks at what types have already
-;; been marked for upgrade so we don't need to generate overly-complicated queries.
-;;
-;; e.g. instead of doing:
-;;
-;; WHERE ((version < 2 AND base_type IN ("type/Integer", "type/BigInteger", "type/Text")) OR
-;; (version < 1 AND base_type IN ("type/Boolean", "type/Integer", "type/BigInteger")))
-;;
-;; we could do:
-;;
-;; WHERE ((version < 2 AND base_type IN ("type/Integer", "type/BigInteger", "type/Text")) OR
-;; (version < 1 AND base_type IN ("type/Boolean")))
-;;
-;; (In the example above, something that is a `type/Integer` or `type/Text` would get upgraded
-;; as long as it's less than version 2; so no need to also check if those types are less than 1, which
-;; would always be the case.)
-;;
-;; This way we can also completely omit adding clauses for versions that have been "eclipsed" by others.
-;; This would keep the SQL query from growing boundlessly as new fingerprint versions are added
-(mu/defn- versions-clauses :- [:maybe [:sequential :any]]
- []
- ;; keep track of all the base types (including descendants) for each version, starting from most recent
- (let [versions+base-types (reverse (sort-by first (seq i/*fingerprint-version->types-that-should-be-re-fingerprinted*)))
- already-seen (atom #{})]
- (for [[version base-types] versions+base-types
- :let [descendants (base-types->descendants base-types)
- not-yet-seen (set/difference descendants @already-seen)]
- ;; if all the descendants of any given version have already been seen, we can skip this clause altogether
- :when (seq not-yet-seen)]
- ;; otherwise record the newly seen types and generate an appropriate clause
- (do
- (swap! already-seen set/union not-yet-seen)
- [:and
- [:< :fingerprint_version version]
- [:in :base_type not-yet-seen]]))))
-
-(def ^:private fields-to-fingerprint-base-clause
- "Base clause to get fields for fingerprinting. When refingerprinting, run as is. When fingerprinting in analysis, only
- look for fields without a fingerprint or whose version can be updated. This clauses is added on
- by [[versions-clauses]]."
- [:and
- [:= :active true]
- [:or
- [:not (app-db/isa :semantic_type :type/PK)]
- [:= :semantic_type nil]]
- [:not-in :visibility_type ["retired" "sensitive"]]
- [:not-in :base_type (conj (app-db/type-keyword->descendants :type/fingerprint-unsupported)
- (u/qualified-name :type/*))]])
-
(def ^:dynamic *refingerprint?*
"Whether we are refingerprinting or doing the normal fingerprinting. Refingerprinting should get fields that already
are analyzed and have fingerprints."
false)
-(mu/defn- honeysql-for-fields-that-need-fingerprint-updating :- [:map
- [:where :any]]
- "Return appropriate WHERE clause for all the Fields whose Fingerprint needs to be re-calculated."
- ([]
- {:where (cond-> fields-to-fingerprint-base-clause
- (not *refingerprint?*) (conj (cons :or (versions-clauses))))})
-
- ([table :- i/TableInstance]
- (sql.helpers/where (honeysql-for-fields-that-need-fingerprint-updating)
- [:= :table_id (u/the-id table)])))
-
;;; +----------------------------------------------------------------------------------------------------------------+
;;; | FINGERPRINTING ALL FIELDS IN A TABLE |
;;; +----------------------------------------------------------------------------------------------------------------+
@@ -202,7 +112,11 @@
id so the selection is stable across syncs. This should include NEW fields that are active and visible."
[table :- i/TableInstance
limit :- ms/PositiveInt]
- (seq (sync.db/fields-where (:where (honeysql-for-fields-that-need-fingerprint-updating table)) limit)))
+ (seq (sync.db/fields-needing-fingerprint-update
+ (u/the-id table)
+ *refingerprint?*
+ i/*fingerprint-version->types-that-should-be-re-fingerprinted*
+ limit)))
(mu/defn- warn-too-many-fields!
"Log that `table` has more fields to fingerprint than fingerprint-max-fields-per-table (`limit`), so only the first
diff --git a/src/metabase/sync/db.clj b/src/metabase/sync/db.clj
index 43622ed757d..08533d0159c 100644
--- a/src/metabase/sync/db.clj
+++ b/src/metabase/sync/db.clj
@@ -2,6 +2,11 @@
"Application database queries for the sync module. Every function here is a direct Toucan 2 call with no additional
logic, so the rest of the module never talks to `toucan2.core` itself."
(:require
+ [clojure.set :as set]
+ [honey.sql :as sql]
+ [metabase.app-db.core :as app-db]
+ [metabase.util :as u]
+ [metabase.util.honey-sql-2 :as h2x]
[metabase.warehouse-schema.models.table :as table]
[toucan2.core :as t2]))
@@ -216,10 +221,61 @@
:table_id :has_field_values]
:id [:in field-ids]))
-(defn fields-where
- "Up to `limit` Fields matching the Honey SQL `where`, ordered by ID."
- [where limit]
- (t2/select :model/Field {:where where, :order-by [[:id :asc]], :limit limit}))
+(defn- base-types->descendants
+ "Given a set of `base-types`, an expanded set including those types and all their descendants in the type
+ hierarchy, as qualified strings so HoneySQL doesn't confuse them for columns."
+ [base-types]
+ (into #{}
+ (comp (mapcat (fn [base-type] (cons base-type (descendants base-type))))
+ (map u/qualified-name))
+ base-types))
+
+(defn- fingerprint-version-clauses
+ "Honey SQL `:or` disjuncts, one per fingerprint version older than the latest that hasn't already been superseded,
+ matching Fields whose `base_type` (or a descendant) was re-fingerprinted at that version. `version->base-types` is
+ a map of fingerprint version to the set of base types re-fingerprinted at that version."
+ [version->base-types]
+ (let [versions+base-types (reverse (sort-by first (seq version->base-types)))
+ already-seen (atom #{})]
+ (into [:or]
+ (keep (fn [[version base-types]]
+ (let [not-yet-seen (set/difference (base-types->descendants base-types) @already-seen)]
+ (when (seq not-yet-seen)
+ (swap! already-seen set/union not-yet-seen)
+ [:and
+ [:< :fingerprint_version version]
+ [:in :base_type not-yet-seen]]))))
+ versions+base-types)))
+
+(def ^:private fields-to-fingerprint-base-clause
+ [:and
+ [:= :active true]
+ [:or
+ [:not (app-db/isa :semantic_type :type/PK)]
+ [:= :semantic_type nil]]
+ [:not-in :visibility_type ["retired" "sensitive"]]
+ [:not-in :base_type (conj (app-db/type-keyword->descendants :type/fingerprint-unsupported)
+ (u/qualified-name :type/*))]])
+
+(defn- needs-fingerprint-update-clause
+ "Honey SQL clause matching Fields whose fingerprint needs to be re-calculated: active, non-`PK`/no-semantic-type,
+ non-retired/sensitive-visibility, non-fingerprint-unsupported `base_type`, and, unless `refingerprint?`, without a
+ fingerprint or whose fingerprint version can be updated per `version->base-types` (a map of fingerprint version to
+ the set of base types that should be re-fingerprinted at that version)."
+ [refingerprint? version->base-types]
+ (cond-> fields-to-fingerprint-base-clause
+ (not refingerprint?) (conj (fingerprint-version-clauses version->base-types))))
+
+(defn fields-needing-fingerprint-update
+ "Up to `limit` active, visible Fields of the Table with `table-id` whose fingerprint needs to be re-calculated,
+ ordered by ID. See [[needs-fingerprint-update-clause]] for the full matching criteria."
+ [table-id refingerprint? version->base-types limit]
+ (t2/select :model/Field
+ {:where [:and
+ [:= :table_id table-id]
+ (needs-fingerprint-update-clause refingerprint? version->base-types)]
+ :order-by [[:id :asc]]
+ :limit limit}))
(defn field-fingerprint
"The fingerprint of the Field with `field-id`."
@@ -345,9 +401,13 @@
(t2/update! :model/Field :id [:in field-ids] {:fingerprint_version fingerprint-version}))
(defn set-table-fields-indexed!
- "Set `database_indexed` of the Fields of the Table with `table-id` to the SQL expression `indexed-expr`."
- [table-id indexed-expr]
- (t2/update! :model/Field {:table_id table-id} {:database_indexed indexed-expr}))
+ "Mark the Fields of the Table with `table-id` whose id is in `indexed-field-ids` as indexed, and all its other
+ Fields as not indexed."
+ [table-id indexed-field-ids]
+ (t2/update! :model/Field {:table_id table-id}
+ {:database_indexed (if (seq indexed-field-ids)
+ [:case [:in :id indexed-field-ids] true :else false]
+ false)}))
(defn set-top-level-fields-indexed!
"Set `database_indexed` of the top-level Fields with `field-ids` to `indexed?`."
@@ -375,10 +435,83 @@
:where [:and sync-tables-clause [:= :db_id database-id]]}]}
{:last_analyzed :%now}))
-(defn execute-one!
- "Run the SQL statement `sql` and return its single result."
- [sql]
- (t2/query-one sql))
+(defn- fk-field-id-subquery
+ "Subquery selecting the (lowest) id of the Field named `column-name` on the Table `[table-schema table-name]` in
+ the Database with `db-id`, excluding Fields with a user-set foreign key target or semantic type. `min` limits the
+ subquery to one result (MySQL disallows `LIMIT` in subqueries), needed because schema/table/column names can be
+ non-unique when lower-cased for some DBs."
+ [db-id table-schema table-name column-name]
+ ^:allow-subquery
+ {:select [[[:min :f.id] :id]]
+ :from [[:metabase_field :f]]
+ :join [[:metabase_table :t] [:= :f.table_id :t.id]]
+ :left-join [[:metabase_field_user_settings :u] [:= :f.id :u.field_id]]
+ :where [:and
+ [:= :u.fk_target_field_id nil]
+ [:= :u.semantic_type nil]
+ [:= :t.db_id db-id]
+ [:= [:lower :f.name] (u/lower-case-en column-name)]
+ [:= [:lower :t.name] (u/lower-case-en table-name)]
+ [:= [:lower :t.schema] (some-> table-schema u/lower-case-en)]
+ [:= :f.active true]
+ [:not= :f.visibility_type "retired"]
+ [:= :t.active true]
+ [:= :t.visibility_type nil]]})
+
+(defn- fk-target-changed-clause
+ "Honey SQL clause true when the Field's `fk_target_field_id` is not already `pk-id-expr`."
+ [pk-id-expr]
+ [:or
+ [:= :f.fk_target_field_id nil]
+ [:not= :f.fk_target_field_id pk-id-expr]])
+
+(defn- mark-fk-statement
+ "`[sql & params]` updating the `fk_target_field_id` of the Field at `[fk-table-schema fk-table-name
+ fk-column-name]` in the Database with `db-id` to the id of the Field at `[pk-table-schema pk-table-name
+ pk-column-name]`, unless it already points there, per the application DB's dialect."
+ [db-id fk-table-schema fk-table-name fk-column-name pk-table-schema pk-table-name pk-column-name]
+ (let [fk-field-id-query (fk-field-id-subquery db-id fk-table-schema fk-table-name fk-column-name)
+ pk-field-id-query (fk-field-id-subquery db-id pk-table-schema pk-table-name pk-column-name)
+ q (case (app-db/db-type)
+ :mysql
+ {:update [:metabase_field :f]
+ :join [[fk-field-id-query :fk] [:= :fk.id :f.id]
+ [pk-field-id-query :pk]
+ (fk-target-changed-clause :pk.id)]
+ :set {:fk_target_field_id :pk.id
+ ;; We need to reset has_field_values when it is auto-list as FKs should not be marked as such
+ :has_field_values [:case [:= :has_field_values "auto-list"] nil :else :has_field_values]
+ :semantic_type "type/FK"}}
+ :postgres
+ {:update [:metabase_field :f]
+ :from [[fk-field-id-query :fk]]
+ :join [[pk-field-id-query :pk] true]
+ :set {:fk_target_field_id :pk.id
+ ;; We need to reset has_field_values when it is auto-list as FKs should not be marked as such
+ :has_field_values [:case [:= :has_field_values "auto-list"] nil :else :has_field_values]
+ :semantic_type "type/FK"}
+ :where [:and
+ [:= :fk.id :f.id]
+ (fk-target-changed-clause :pk.id)]}
+ :h2
+ {:update [:metabase_field :f]
+ :set {:fk_target_field_id pk-field-id-query
+ ;; We need to reset has_field_values when it is auto-list as FKs should not be marked as such
+ :has_field_values [:case [:= :has_field_values "auto-list"] nil :else :has_field_values]
+ :semantic_type "type/FK"}
+ :where [:and
+ [:= :f.id fk-field-id-query]
+ [:not= pk-field-id-query nil]
+ (fk-target-changed-clause pk-field-id-query)]})]
+ (sql/format q :dialect (app-db/quoting-style (app-db/db-type)))))
+
+(defn mark-fk!
+ "Set the `fk_target_field_id` of the Field at `[fk-table-schema fk-table-name fk-column-name]` in the Database with
+ `db-id` to the id of the Field at `[pk-table-schema pk-table-name pk-column-name]`, unless it already points there.
+ Returns 1 if a Field was updated, 0 otherwise."
+ [db-id fk-table-schema fk-table-name fk-column-name pk-table-schema pk-table-name pk-column-name]
+ (t2/query-one (mark-fk-statement db-id fk-table-schema fk-table-name fk-column-name
+ pk-table-schema pk-table-name pk-column-name)))
;;; ---------------------------------------------- FieldValues ----------------------------------------------
@@ -387,13 +520,19 @@
[field-id]
(t2/exists? :model/FieldValues :field_id field-id))
+(defn- before-max-age-value
+ "Honey SQL `[:< …]` value expression matching a timestamp more than `max-age-days` days before now."
+ [max-age-days]
+ [:< (h2x/add-interval-honeysql-form (app-db/db-type) :%now (- max-age-days) :day)])
+
(defn advanced-field-values-count-before
- "The number of FieldValues of `types` for the Field with `field-id` created before the SQL expression
- `created-before`."
- [field-id types created-before]
- (t2/count :model/FieldValues :field_id field-id :type [:in types] :created_at [:< created-before]))
+ "The number of FieldValues of `types` for the Field with `field-id` created more than `max-age-days` days ago."
+ [field-id types max-age-days]
+ (t2/count :model/FieldValues :field_id field-id :type [:in types]
+ :created_at (before-max-age-value max-age-days)))
(defn delete-advanced-field-values-before!
- "Delete the FieldValues of `types` for the Field with `field-id` created before the SQL expression `created-before`."
- [field-id types created-before]
- (t2/delete! :model/FieldValues :field_id field-id :type [:in types] :created_at [:< created-before]))
+ "Delete the FieldValues of `types` for the Field with `field-id` created more than `max-age-days` days ago."
+ [field-id types max-age-days]
+ (t2/delete! :model/FieldValues :field_id field-id :type [:in types]
+ :created_at (before-max-age-value max-age-days)))
diff --git a/src/metabase/sync/field_values.clj b/src/metabase/sync/field_values.clj
index 88148a61975..80533dfc333 100644
--- a/src/metabase/sync/field_values.clj
+++ b/src/metabase/sync/field_values.clj
@@ -2,7 +2,6 @@
"Logic for updating FieldValues for fields in a database."
(:require
[java-time.api :as t]
- [metabase.app-db.core :as mdb]
[metabase.driver :as driver]
[metabase.driver.util :as driver.u]
[metabase.sync.db :as sync.db]
@@ -268,14 +267,10 @@
(defn- delete-expired-advanced-field-values-for-field!
[field]
(sync-util/with-error-handling (format "Error deleting expired advanced field values for %s" (sync-util/name-for-logging field))
- (let [types field-values/advanced-field-values-types
- cutoff ((requiring-resolve 'metabase.util.honey-sql-2/add-interval-honeysql-form)
- (mdb/db-type)
- :%now
- (- (t/as field-values/advanced-field-values-max-age :days))
- :day)
- rows-count (sync.db/advanced-field-values-count-before (:id field) types cutoff)]
- (sync.db/delete-advanced-field-values-before! (:id field) types cutoff)
+ (let [types field-values/advanced-field-values-types
+ max-age-days (t/as field-values/advanced-field-values-max-age :days)
+ rows-count (sync.db/advanced-field-values-count-before (:id field) types max-age-days)]
+ (sync.db/delete-advanced-field-values-before! (:id field) types max-age-days)
rows-count)))
(mu/defn delete-expired-advanced-field-values-for-table!
diff --git a/src/metabase/sync/sync_metadata/fks.clj b/src/metabase/sync/sync_metadata/fks.clj
index b337082ac05..8556bbf3665 100644
--- a/src/metabase/sync/sync_metadata/fks.clj
+++ b/src/metabase/sync/sync_metadata/fks.clj
@@ -1,8 +1,6 @@
(ns metabase.sync.sync-metadata.fks
"Logic for updating FK properties of Fields from metadata fetched from a physical DB."
(:require
- [honey.sql :as sql]
- [metabase.app-db.core :as mdb]
[metabase.driver.util :as driver.u]
[metabase.sync.db :as sync.db]
[metabase.sync.fetch-metadata :as fetch-metadata]
@@ -13,87 +11,13 @@
[metabase.util.malli :as mu]
[metabase.warehouse-schema.models.table :as table]))
-(defn ^:private mark-fk-sql
- "Returns [sql & params] for [[mark-fk!]] according to the application DB's dialect."
- [db-id {:keys [fk-table-name
- fk-table-schema
- fk-column-name
- pk-table-name
- pk-table-schema
- pk-column-name]}]
- (let [field-id-query (fn [db-id table-schema table-name column-name]
- ^:allow-subquery
- {:select [[[:min :f.id] :id]]
- ;; Cal 2024-03-04: We use `min` to limit this subquery to one result (limit 1 isn't allowed
- ;; in subqueries in MySQL) because it's possible for schema, table, or column names to be
- ;; non-unique when lower-cased for some DBs. We have been doing case-insensitive matching
- ;; since #5510 so this preserves behaviour to avoid possible regressions.
- ;; It's possible this is to avoid
- :from [[:metabase_field :f]]
- :join [[:metabase_table :t] [:= :f.table_id :t.id]]
- :left-join [[:metabase_field_user_settings :u] [:= :f.id :u.field_id]]
- :where [:and
- ;; ensure we are not overriding user-set fks
- [:= :u.fk_target_field_id nil]
- [:= :u.semantic_type nil]
- [:= :t.db_id db-id]
- [:= [:lower :f.name] (u/lower-case-en column-name)]
- [:= [:lower :t.name] (u/lower-case-en table-name)]
- [:= [:lower :t.schema] (some-> table-schema u/lower-case-en)]
- [:= :f.active true]
- [:not= :f.visibility_type "retired"]
- [:= :t.active true]
- [:= :t.visibility_type nil]]})
- fk-field-id-query (field-id-query db-id fk-table-schema fk-table-name fk-column-name)
- pk-field-id-query (field-id-query db-id pk-table-schema pk-table-name pk-column-name)
-
- ;; Only update if either:
- ;; - fk_target_field_id is NULL and the new target is not NULL
- ;; - fk_target_field_id is not NULL but the new target is different and not NULL
- valid-condition
- (fn [k]
- [:or
- [:= :f.fk_target_field_id nil]
- [:not= :f.fk_target_field_id k]])
-
- q (case (mdb/db-type)
- :mysql
- {:update [:metabase_field :f]
- :join [[fk-field-id-query :fk] [:= :fk.id :f.id]
- [pk-field-id-query :pk]
- (valid-condition :pk.id)]
- :set {:fk_target_field_id :pk.id
- ;; We need to reset has_field_values when it is auto-list as FKs should not be marked as such
- :has_field_values [:case [:= :has_field_values "auto-list"] nil :else :has_field_values]
- :semantic_type "type/FK"}}
- :postgres
- {:update [:metabase_field :f]
- :from [[fk-field-id-query :fk]]
- :join [[pk-field-id-query :pk] true]
- :set {:fk_target_field_id :pk.id
- ;; We need to reset has_field_values when it is auto-list as FKs should not be marked as such
- :has_field_values [:case [:= :has_field_values "auto-list"] nil :else :has_field_values]
- :semantic_type "type/FK"}
- :where [:and
- [:= :fk.id :f.id]
- (valid-condition :pk.id)]}
- :h2
- {:update [:metabase_field :f]
- :set {:fk_target_field_id pk-field-id-query
- ;; We need to reset has_field_values when it is auto-list as FKs should not be marked as such
- :has_field_values [:case [:= :has_field_values "auto-list"] nil :else :has_field_values]
- :semantic_type "type/FK"}
- :where [:and
- [:= :f.id fk-field-id-query]
- [:not= pk-field-id-query nil]
- (valid-condition pk-field-id-query)]})]
- (sql/format q :dialect (mdb/quoting-style (mdb/db-type)))))
-
(mu/defn- mark-fk!
"Updates the `fk_target_field_id` of a Field. Returns 1 if the Field was successfully updated, 0 otherwise."
[database :- i/DatabaseInstance
metadata :- i/FKMetadataEntry]
- (u/prog1 (sync.db/execute-one! (mark-fk-sql (:id database) metadata))
+ (u/prog1 (sync.db/mark-fk! (:id database)
+ (:fk-table-schema metadata) (:fk-table-name metadata) (:fk-column-name metadata)
+ (:pk-table-schema metadata) (:pk-table-name metadata) (:pk-column-name metadata))
(when (= <> 1)
(log/info (u/format-color 'cyan "Marking foreign key from %s %s -> %s %s"
(sync-util/table-name-for-logging :name (:fk-table-name metadata)
diff --git a/src/metabase/sync/sync_metadata/indexes.clj b/src/metabase/sync/sync_metadata/indexes.clj
index 466411f8519..7cd2da73ae7 100644
--- a/src/metabase/sync/sync_metadata/indexes.clj
+++ b/src/metabase/sync/sync_metadata/indexes.clj
@@ -38,10 +38,7 @@
(doseq [field-id adding]
(log/infof "Marking Field %d as indexed" field-id))
(if (or (seq adding) (seq removing))
- (do (sync.db/set-table-fields-indexed! (:id table)
- (if (seq indexed-field-ids)
- [:case [:in :id indexed-field-ids] true :else false]
- false))
+ (do (sync.db/set-table-fields-indexed! (:id table) indexed-field-ids)
{:total-indexes (count indexed-field-ids)
:added-indexes (count adding)
:removed-indexes (count removing)})
diff --git a/src/metabase/typed_schemas/db.clj b/src/metabase/typed_schemas/db.clj
index 8c88db336b9..71a15fc597a 100644
--- a/src/metabase/typed_schemas/db.clj
+++ b/src/metabase/typed_schemas/db.clj
@@ -2,6 +2,7 @@
"Application database queries for the typed schemas module. Every function here is a direct Toucan 2 call with no
additional logic, so the rest of the module never talks to `toucan2.core` itself."
(:require
+ [metabase.collections.models.collection :as collection]
[toucan2.core :as t2]))
(defn destination-database-ids
@@ -9,10 +10,28 @@
[database-ids]
(t2/select-fn-set :id :model/Database :id [:in database-ids] :router_database_id [:not= nil]))
+(defn- scope-filter-clause
+ "Compiles a resolved scope into a Honey SQL where-clause conjunct: a nil scope means unscoped (no clause), an
+ empty scope matches nothing."
+ [scope-ids column]
+ (when scope-ids
+ (if (seq scope-ids)
+ [:in column scope-ids]
+ ;; no row has id -1: a resolved-but-empty scope matches no rows
+ [:= column -1])))
+
(defn cards-ordered-by-name
- "The Cards matching the Honey SQL `where` clause, in name then id order."
- [where]
- (t2/select :model/Card {:where where, :order-by [[:name :asc] [:id :asc]]}))
+ "The readable, non-archived Cards of `card-type` among `database-ids` and/or `collection-ids` (either nil for
+ unscoped), in name then id order."
+ [card-type database-ids collection-ids]
+ (t2/select :model/Card
+ {:where [:and
+ [:= :type (name card-type)]
+ [:= :archived false]
+ (collection/visible-collection-filter-clause :collection_id)
+ (scope-filter-clause database-ids :database_id)
+ (scope-filter-clause collection-ids :collection_id)]
+ :order-by [[:name :asc] [:id :asc]]}))
(defn field-ids-and-table-ids
"The id and Table id of the Fields with `field-ids`."
@@ -42,16 +61,6 @@
[field-id]
(t2/select-one-fn :table_id :model/Field :id field-id))
-(defn- scope-filter-clause
- "Compiles a resolved scope into a Honey SQL where-clause conjunct: a nil scope means unscoped (no clause), an
- empty scope matches nothing."
- [scope-ids column]
- (when scope-ids
- (if (seq scope-ids)
- [:in column scope-ids]
- ;; no row has id -1: a resolved-but-empty scope matches no rows
- [:= column -1])))
-
(defn active-tables-in-scope
"The active Tables among `database-ids` and/or `table-ids` (either nil for unscoped), in name then id order."
[database-ids table-ids]
diff --git a/src/metabase/typed_schemas/schema/common.clj b/src/metabase/typed_schemas/schema/common.clj
index 1fb0c4b4666..105fcea3cb8 100644
--- a/src/metabase/typed_schemas/schema/common.clj
+++ b/src/metabase/typed_schemas/schema/common.clj
@@ -2,7 +2,6 @@
"Shared typed-schema source helpers."
(:require
[medley.core :as m]
- [metabase.collections.models.collection :as collection]
[metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
[metabase.metabot.core :as metabot]
@@ -11,23 +10,6 @@
(set! *warn-on-reflection* true)
-(defn scope-filter-clause
- "Compiles a resolved scope (see [[metabase.typed-schemas.scope]]) into a
- Honey SQL where-clause conjunct: a nil scope means unscoped (no clause), an
- empty scope matches nothing.
-
- This is the one place the nil-vs-empty semantics of scopes are enforced when
- querying. Do not replace call sites with `(seq ...)` guards — that would
- turn \"resolved to nothing\" into \"unscoped\" and select everything.
- Compiling the empty scope to an always-false clause (rather than skipping
- the query) keeps queries with several scope filters composable."
- [scope-ids column]
- (when scope-ids
- (if (seq scope-ids)
- [:in column scope-ids]
- ;; no row has id -1: a resolved-but-empty scope matches no rows
- [:= column -1])))
-
(defn destination-db-ids
"Returns the subset of `db-ids` that back a destination (routed) database.
@@ -45,13 +27,7 @@
the same visibility, archived, database and collection filters, and exclude cards backed by a
destination (routed) database -- see [[destination-db-ids]]."
[card-type database-ids collection-ids]
- (let [cards (->> (typed-schemas.db/cards-ordered-by-name
- (cond-> [:and
- [:= :type (name card-type)]
- [:= :archived false]
- (collection/visible-collection-filter-clause :collection_id)]
- database-ids (conj (scope-filter-clause database-ids :database_id))
- collection-ids (conj (scope-filter-clause collection-ids :collection_id))))
+ (let [cards (->> (typed-schemas.db/cards-ordered-by-name card-type database-ids collection-ids)
(filter mi/can-read?))
destination-ids (destination-db-ids (into #{} (keep :database_id) cards))]
(if (seq destination-ids)
diff --git a/src/metabase/warehouse_schema/db.clj b/src/metabase/warehouse_schema/db.clj
index dc0f6395471..8de4de6e3ab 100644
--- a/src/metabase/warehouse_schema/db.clj
+++ b/src/metabase/warehouse_schema/db.clj
@@ -4,6 +4,7 @@
methods, and transactions."
(:require
[metabase.app-db.core :as app-db]
+ [metabase.models.db :as models.db]
[toucan2.core :as t2]))
(def field-order-rule
@@ -17,22 +18,11 @@
[field-id]
(t2/select-one :model/Field :id field-id))
-(defn- field-in-path-query
- [table-id [field & rest]]
- (when field
- ^:allow-subquery {:from [:metabase_field]
- :select [:id]
- :where [:and
- [:= :table_id table-id]
- [:= :name field]
- [:= :parent_id (field-in-path-query table-id rest)]]}))
-
(defn field-in-path
"The Field named by the last of `field-names` (each nested inside the previous, bottom-most first) under
- `table-id`, or nil."
+ `table-id`, or nil. See `metabase.models.db/field-in-path`, which owns the shared query."
[table-id field-names]
- (when (seq field-names)
- (t2/select-one :model/Field (field-in-path-query table-id field-names))))
+ (models.db/field-in-path table-id field-names))
(defn fields
"The Fields with `field-ids`."
diff --git a/src/metabase/warehouses_rest/api.clj b/src/metabase/warehouses_rest/api.clj
index f139c6a4b38..59410948a09 100644
--- a/src/metabase/warehouses_rest/api.clj
+++ b/src/metabase/warehouses_rest/api.clj
@@ -8,7 +8,6 @@
[metabase.api.macros :as api.macros]
[metabase.app-db.core :as mdb]
[metabase.classloader.core :as classloader]
- [metabase.collections.models.collection :as collection]
[metabase.config.core :as config]
[metabase.database-routing.core :as database-routing]
[metabase.driver :as driver]
@@ -205,7 +204,7 @@
(mu/defn- source-query-cards
"Fetch the Cards that can be used as source queries (e.g. presented as virtual tables)."
[card-type :- ::queries.schema/card-type
- & {:keys [additional-constraints xform], :or {xform identity}}]
+ & {:keys [collection-scope xform], :or {xform identity}}]
(when-let [ids-of-dbs-that-support-source-queries (not-empty (ids-of-dbs-that-support-source-queries))]
(transduce
(comp (map (partial mi/do-after-select :model/Card))
@@ -214,14 +213,7 @@
(completing conj #(t2/hydrate % :collection :metrics))
[]
(warehouses-rest.db/source-query-cards-reducible
- (into [:and
- [:not= :result_metadata nil]
- [:= :archived false]
- ;; always return metrics for now
- [:in :type [(u/qualified-name card-type) "metric"]]
- [:in :database_id ids-of-dbs-that-support-source-queries]
- (collection/visible-collection-filter-clause)]
- additional-constraints)))))
+ card-type ids-of-dbs-that-support-source-queries collection-scope))))
(mu/defn- source-query-cards-exist?
"Truthy if a single Card that can be used as a source query exists."
@@ -316,19 +308,8 @@
user-info {:user-id api/*current-user-id*
:is-superuser? (mi/superuser?)
:is-data-analyst? api/*is-data-analyst?*}
- base-where [:and
- [:= :is_stub false]
- (when-not include-analytics?
- [:= :is_audit false])
- (if filter-on-router-database-id
- [:= :router_database_id router-database-id]
- [:= :router_database_id nil])]
- where-clause (if filter-by-data-access?
- [:and base-where [:or (:clause (mi/visible-filter-clause :model/Database :id user-info {:perms/create-queries :query-builder}))
- (:clause (mi/visible-filter-clause :model/Database :id user-info {:perms/manage-database :yes}))
- (:clause (mi/visible-filter-clause :model/Database :id user-info {:perms/manage-table-metadata :yes}))]]
- base-where)
- dbs (warehouses-rest.db/databases-where where-clause)
+ dbs (warehouses-rest.db/databases-where user-info filter-by-data-access? filter-on-router-database-id
+ include-analytics?)
;; everything below walks the list one database at a time
_ (perms/prime-database-perms-cache {:db-ids (into #{} (map :id) dbs)})]
(cond-> (-> dbs add-native-perms-info add-transforms-perms-info)
@@ -1326,10 +1307,6 @@
(map :schema (f (map (fn [s] {:db_id id :schema s}) schemas)))
schemas)
(filter (partial can-read-schema? id) schemas)))
- clauses (cond-> []
- ;; a non-nil value means Table is hidden --
- ;; see [[metabase.warehouse-schema.models.table/visibility-types]]
- (not include-hidden?) (conj [:= :visibility_type nil]))
;; For can-query? and can-write-metadata?, we need to filter based on tables in each schema
filter-schemas-by-tables (fn [schemas]
(if (or can-query? can-write-metadata?)
@@ -1342,11 +1319,7 @@
(filter #(contains? allowed-schemas %) schemas))
schemas))]
(warehouses/get-database id {:include-editable-data-model? include-editable-data-model?})
- (->> (warehouses-rest.db/active-table-schemas id
- (merge
- {:order-by [[:%lower.schema :asc]]}
- (when clauses
- {:where (into [:and] clauses)})))
+ (->> (warehouses-rest.db/active-table-schemas id include-hidden?)
filter-schemas
filter-schemas-by-tables
;; for `nil` schemas return the empty string
@@ -1520,9 +1493,9 @@
(when (lib-be/enable-nested-queries)
(->> (source-query-cards
:question
- :additional-constraints [(if (= schema (schema.table/root-collection-schema-name))
- [:= :collection_id nil]
- [:in :collection_id (api/check-404 (not-empty (warehouses-rest.db/collection-ids-named schema)))])])
+ :collection-scope (if (= schema (schema.table/root-collection-schema-name))
+ :root
+ (api/check-404 (not-empty (warehouses-rest.db/collection-ids-named schema)))))
(map schema.table/card->virtual-table))))
;; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to
@@ -1557,9 +1530,9 @@
(when (lib-be/enable-nested-queries)
(->> (source-query-cards
:model
- :additional-constraints [(if (= schema (schema.table/root-collection-schema-name))
- [:= :collection_id nil]
- [:in :collection_id (api/check-404 (not-empty (warehouses-rest.db/collection-ids-named schema)))])])
+ :collection-scope (if (= schema (schema.table/root-collection-schema-name))
+ :root
+ (api/check-404 (not-empty (warehouses-rest.db/collection-ids-named schema)))))
(map schema.table/card->virtual-table))))
;;; -------------------------------- GET /api/database/:id/settings-available ------------------------------------
diff --git a/src/metabase/warehouses_rest/db.clj b/src/metabase/warehouses_rest/db.clj
index 02defdc708f..d8f2fefbf23 100644
--- a/src/metabase/warehouses_rest/db.clj
+++ b/src/metabase/warehouses_rest/db.clj
@@ -4,6 +4,8 @@
(:require
[clojure.string :as str]
[metabase.app-db.core :as mdb]
+ [metabase.collections.models.collection :as collection]
+ [metabase.models.interface :as mi]
[metabase.util :as u]
[metabase.util.honey-sql-2 :as h2x]
[toucan2.core :as t2]))
@@ -34,9 +36,11 @@
(t2/select [:model/Database :id :engine]))
(defn source-query-cards-reducible
- "A reducible of the Cards, with their moderation status, matching the Honey SQL `where` clause, in
- case-insensitive name order."
- [where]
+ "A reducible of the Cards of `card-type` (also including \"metric\" Cards) in the Databases with `database-ids`
+ visible to the current user that can be used as source queries, with their moderation status, in case-insensitive
+ name order. `collection-scope` further restricts by collection: `nil` applies no collection restriction, `:root`
+ restricts to Cards with no collection, and a collection of ids restricts to Cards in those collections."
+ [card-type database-ids collection-scope]
(t2/reducible-query {:select [:name :description :database_id :dataset_query :id :collection_id
:result_metadata :type :source_card_id :card_schema
[^:allow-subquery {:select [:status]
@@ -49,14 +53,41 @@
:limit 1}
:moderated_status]]
:from [:report_card]
- :where where
+ :where [:and
+ [:not= :result_metadata nil]
+ [:= :archived false]
+ [:in :type [(u/qualified-name card-type) "metric"]]
+ [:in :database_id database-ids]
+ (cond
+ (nil? collection-scope) nil
+ (= collection-scope :root) [:= :collection_id nil]
+ :else [:in :collection_id collection-scope])
+ (collection/visible-collection-filter-clause)]
:order-by [[:%lower.name :asc]]}))
(defn databases-where
- "The Databases matching the Honey SQL `where` clause, in name then engine order."
- [where]
- (t2/select :model/Database {:order-by [:%lower.name :%lower.engine]
- :where where}))
+ "The Databases visible to the user described by `user-info` (a map of `:user-id`/`:is-superuser?`/
+ `:is-data-analyst?`), in name then engine order. Excludes stub Databases and, unless `include-analytics?`, the
+ audit Database. Restricted to Databases routed from `router-database-id` when given, otherwise to non-routed
+ Databases. When `filter-by-data-access?` is true, further restricted to Databases the user can query, manage, or
+ edit the metadata of."
+ [user-info filter-by-data-access? router-database-id include-analytics?]
+ (let [base-where [:and
+ [:= :is_stub false]
+ (when-not include-analytics?
+ [:= :is_audit false])
+ (if router-database-id
+ [:= :router_database_id router-database-id]
+ [:= :router_database_id nil])]
+ where (if filter-by-data-access?
+ [:and base-where
+ [:or
+ (:clause (mi/visible-filter-clause :model/Database :id user-info {:perms/create-queries :query-builder}))
+ (:clause (mi/visible-filter-clause :model/Database :id user-info {:perms/manage-database :yes}))
+ (:clause (mi/visible-filter-clause :model/Database :id user-info {:perms/manage-table-metadata :yes}))]]
+ base-where)]
+ (t2/select :model/Database {:order-by [:%lower.name :%lower.engine]
+ :where where})))
(defn database-exists?
"Whether a Database with `database-id` exists."
@@ -222,9 +253,16 @@
(t2/select :model/Table :db_id database-id :active true))
(defn active-table-schemas
- "The schemas of the active Tables of the Database with `database-id` also selected by the Honey SQL `query`."
- [database-id query]
- (t2/select-fn-set :schema :model/Table :db_id database-id :active true query))
+ "The distinct schemas of the active Tables of the Database with `database-id`, in schema order. When
+ `include-hidden?` is false, restricted to Tables with no `visibility_type` (a non-nil value means the Table is
+ hidden -- see [[metabase.warehouse-schema.models.table/visibility-types]])."
+ [database-id include-hidden?]
+ (let [clauses (cond-> []
+ (not include-hidden?) (conj [:= :visibility_type nil]))]
+ (t2/select-fn-set :schema :model/Table :db_id database-id :active true
+ (merge {:order-by [[:%lower.schema :asc]]}
+ (when clauses
+ {:where (into [:and] clauses)})))))
(defn active-tables-in-schema
"The active Tables in `schema` of the Database with `database-id`, in display name order."
diff --git a/test/metabase/sync/analyze/fingerprint_test.clj b/test/metabase/sync/analyze/fingerprint_test.clj
index 23bf1b596e5..fc811caf911 100644
--- a/test/metabase/sync/analyze/fingerprint_test.clj
+++ b/test/metabase/sync/analyze/fingerprint_test.clj
@@ -6,6 +6,7 @@
[metabase.app-db.core :as app-db]
[metabase.query-processor :as qp]
[metabase.sync.analyze.fingerprint :as sync.fingerprint]
+ [metabase.sync.db :as sync.db]
[metabase.sync.interface :as i]
[metabase.test :as mt]
[metabase.test.data :as data]
@@ -23,9 +24,9 @@
;; Check that our `base-types->descendants` function properly returns a set of descendants including parent type
(deftest ^:parallel base-type->descendats-test
(is (= #{"type/URL" "type/ImageURL" "type/AvatarURL"}
- (#'sync.fingerprint/base-types->descendants #{:type/URL})))
+ (#'sync.db/base-types->descendants #{:type/URL})))
(is (= #{"type/ImageURL" "type/AvatarURL"}
- (#'sync.fingerprint/base-types->descendants #{:type/ImageURL :type/AvatarURL}))))
+ (#'sync.db/base-types->descendants #{:type/ImageURL :type/AvatarURL}))))
(def ^:private skip-fingerprint-base-types
(into #{"type/*"
@@ -44,108 +45,99 @@
:type/Large
:type/fingerprint-unsupported]))
-(deftest ^:parallel honeysql-for-fields-that-need-fingerprint-updating-test
- (testing (str "Make sure we generate the correct HoneySQL WHERE clause based on whatever is in "
- "`*fingerprint-version->types-that-should-be-re-fingerprinted*`")
- (is (= {:where
- [:and
- [:= :active true]
- [:or
- [:not (app-db/isa :semantic_type :type/PK)]
- [:= :semantic_type nil]]
- [:not-in :visibility_type ["retired" "sensitive"]]
- [:not-in :base_type skip-fingerprint-base-types]
- [:or
- [:and
- [:< :fingerprint_version 1]
- [:in :base_type #{"type/URL" "type/ImageURL" "type/AvatarURL"}]]]]}
- (binding [i/*fingerprint-version->types-that-should-be-re-fingerprinted* {1 #{:type/URL}}]
- (#'sync.fingerprint/honeysql-for-fields-that-need-fingerprint-updating))))))
-
-(deftest ^:parallel honeysql-for-fields-that-need-fingerprint-updating-test-2
- (is (= {:where
- [:and
- [:= :active true]
- [:or
- [:not (app-db/isa :semantic_type :type/PK)]
- [:= :semantic_type nil]]
- [:not-in :visibility_type ["retired" "sensitive"]]
- [:not-in :base_type skip-fingerprint-base-types]
- [:or
- [:and
- [:< :fingerprint_version 2]
- [:in :base_type #{"type/Decimal" "type/Latitude" "type/Longitude" "type/Coordinate" "type/Currency" "type/Float"
- "type/Share" "type/Income" "type/Price" "type/Discount" "type/GrossMargin" "type/Cost" "type/Percentage"}]]
- [:and
- [:< :fingerprint_version 1]
- [:in :base_type #{"type/ImageURL" "type/AvatarURL"}]]]]}
- (binding [i/*fingerprint-version->types-that-should-be-re-fingerprinted* {1 #{:type/ImageURL :type/AvatarURL}
- 2 #{:type/Float}}]
- (#'sync.fingerprint/honeysql-for-fields-that-need-fingerprint-updating)))))
-
-(deftest ^:parallel honeysql-for-fields-that-need-fingerprint-updating-test-3
+(deftest ^:parallel needs-fingerprint-update-clause-test
+ (testing (str "Make sure we generate the correct HoneySQL WHERE clause based on whatever `version->base-types` "
+ "is passed in")
+ (is (= [:and
+ [:= :active true]
+ [:or
+ [:not (app-db/isa :semantic_type :type/PK)]
+ [:= :semantic_type nil]]
+ [:not-in :visibility_type ["retired" "sensitive"]]
+ [:not-in :base_type skip-fingerprint-base-types]
+ [:or
+ [:and
+ [:< :fingerprint_version 1]
+ [:in :base_type #{"type/URL" "type/ImageURL" "type/AvatarURL"}]]]]
+ (#'sync.db/needs-fingerprint-update-clause false {1 #{:type/URL}})))))
+
+(deftest ^:parallel needs-fingerprint-update-clause-test-2
+ (is (= [:and
+ [:= :active true]
+ [:or
+ [:not (app-db/isa :semantic_type :type/PK)]
+ [:= :semantic_type nil]]
+ [:not-in :visibility_type ["retired" "sensitive"]]
+ [:not-in :base_type skip-fingerprint-base-types]
+ [:or
+ [:and
+ [:< :fingerprint_version 2]
+ [:in :base_type #{"type/Decimal" "type/Latitude" "type/Longitude" "type/Coordinate" "type/Currency" "type/Float"
+ "type/Share" "type/Income" "type/Price" "type/Discount" "type/GrossMargin" "type/Cost" "type/Percentage"}]]
+ [:and
+ [:< :fingerprint_version 1]
+ [:in :base_type #{"type/ImageURL" "type/AvatarURL"}]]]]
+ (#'sync.db/needs-fingerprint-update-clause false {1 #{:type/ImageURL :type/AvatarURL}
+ 2 #{:type/Float}}))))
+
+(deftest ^:parallel needs-fingerprint-update-clause-test-3
(testing "our SQL generation code is clever enough to remove version checks when a newer version completely eclipses them"
- (is (= {:where
- [:and
- [:= :active true]
- [:or
- [:not (app-db/isa :semantic_type :type/PK)]
- [:= :semantic_type nil]]
- [:not-in :visibility_type ["retired" "sensitive"]]
- [:not-in :base_type skip-fingerprint-base-types]
- [:or
- [:and
- [:< :fingerprint_version 2]
- [:in :base_type #{"type/Decimal" "type/Latitude" "type/Longitude" "type/Coordinate" "type/Currency" "type/Float"
- "type/Share" "type/Income" "type/Price" "type/Discount" "type/GrossMargin" "type/Cost" "type/Percentage"}]]
- ;; no type/Float stuff should be included for 1
- [:and
- [:< :fingerprint_version 1]
- [:in :base_type #{"type/URL" "type/ImageURL" "type/AvatarURL"}]]]]}
- (binding [i/*fingerprint-version->types-that-should-be-re-fingerprinted* {1 #{:type/Float :type/URL}
- 2 #{:type/Float}}]
- (#'sync.fingerprint/honeysql-for-fields-that-need-fingerprint-updating))))))
-
-(deftest ^:parallel honeysql-for-fields-that-need-fingerprint-updating-test-4
+ (is (= [:and
+ [:= :active true]
+ [:or
+ [:not (app-db/isa :semantic_type :type/PK)]
+ [:= :semantic_type nil]]
+ [:not-in :visibility_type ["retired" "sensitive"]]
+ [:not-in :base_type skip-fingerprint-base-types]
+ [:or
+ [:and
+ [:< :fingerprint_version 2]
+ [:in :base_type #{"type/Decimal" "type/Latitude" "type/Longitude" "type/Coordinate" "type/Currency" "type/Float"
+ "type/Share" "type/Income" "type/Price" "type/Discount" "type/GrossMargin" "type/Cost" "type/Percentage"}]]
+ ;; no type/Float stuff should be included for 1
+ [:and
+ [:< :fingerprint_version 1]
+ [:in :base_type #{"type/URL" "type/ImageURL" "type/AvatarURL"}]]]]
+ (#'sync.db/needs-fingerprint-update-clause false {1 #{:type/Float :type/URL}
+ 2 #{:type/Float}})))))
+
+(deftest ^:parallel needs-fingerprint-update-clause-test-4
(testing "our SQL generation code is also clever enough to completely skip completely eclipsed versions"
- (is (= {:where
- [:and
- [:= :active true]
- [:or
- [:not (app-db/isa :semantic_type :type/PK)]
- [:= :semantic_type nil]]
- [:not-in :visibility_type ["retired" "sensitive"]]
- [:not-in :base_type skip-fingerprint-base-types]
- [:or
- [:and
- [:< :fingerprint_version 4]
- [:in :base_type #{"type/Decimal" "type/Latitude" "type/Longitude" "type/Coordinate" "type/Currency" "type/Float"
- "type/Share" "type/Income" "type/Price" "type/Discount" "type/GrossMargin" "type/Cost" "type/Percentage"}]]
- [:and
- [:< :fingerprint_version 3]
- [:in :base_type #{"type/URL" "type/ImageURL" "type/AvatarURL"}]]
- ;; version 2 can be eliminated completely since everything relevant there is included in 4
- ;; The only things that should go in 1 should be `:type/City` since `:type/Coordinate` is included in 4
- [:and
- [:< :fingerprint_version 1]
- [:in :base_type #{"type/City"}]]]]}
- (binding [i/*fingerprint-version->types-that-should-be-re-fingerprinted* {1 #{:type/Coordinate :type/City}
- 2 #{:type/Coordinate}
- 3 #{:type/URL}
- 4 #{:type/Float}}]
- (#'sync.fingerprint/honeysql-for-fields-that-need-fingerprint-updating))))))
-
-(deftest ^:parallel honeysql-for-fields-that-need-fingerprint-updating-test-5
+ (is (= [:and
+ [:= :active true]
+ [:or
+ [:not (app-db/isa :semantic_type :type/PK)]
+ [:= :semantic_type nil]]
+ [:not-in :visibility_type ["retired" "sensitive"]]
+ [:not-in :base_type skip-fingerprint-base-types]
+ [:or
+ [:and
+ [:< :fingerprint_version 4]
+ [:in :base_type #{"type/Decimal" "type/Latitude" "type/Longitude" "type/Coordinate" "type/Currency" "type/Float"
+ "type/Share" "type/Income" "type/Price" "type/Discount" "type/GrossMargin" "type/Cost" "type/Percentage"}]]
+ [:and
+ [:< :fingerprint_version 3]
+ [:in :base_type #{"type/URL" "type/ImageURL" "type/AvatarURL"}]]
+ ;; version 2 can be eliminated completely since everything relevant there is included in 4
+ ;; The only things that should go in 1 should be `:type/City` since `:type/Coordinate` is included in 4
+ [:and
+ [:< :fingerprint_version 1]
+ [:in :base_type #{"type/City"}]]]]
+ (#'sync.db/needs-fingerprint-update-clause false {1 #{:type/Coordinate :type/City}
+ 2 #{:type/Coordinate}
+ 3 #{:type/URL}
+ 4 #{:type/Float}})))))
+
+(deftest ^:parallel needs-fingerprint-update-clause-test-5
(testing "when refingerprinting doesn't check for versions"
- (is (= {:where [:and
- [:= :active true]
- [:or
- [:not (app-db/isa :semantic_type :type/PK)]
- [:= :semantic_type nil]]
- [:not-in :visibility_type ["retired" "sensitive"]]
- [:not-in :base_type skip-fingerprint-base-types]]}
- (binding [sync.fingerprint/*refingerprint?* true]
- (#'sync.fingerprint/honeysql-for-fields-that-need-fingerprint-updating))))))
+ (is (= [:and
+ [:= :active true]
+ [:or
+ [:not (app-db/isa :semantic_type :type/PK)]
+ [:= :semantic_type nil]]
+ [:not-in :visibility_type ["retired" "sensitive"]]
+ [:not-in :base_type skip-fingerprint-base-types]]
+ (#'sync.db/needs-fingerprint-update-clause true {1 #{:type/URL}})))))
;; Make sure that the above functions are used correctly to determine which Fields get (re-)fingerprinted
(defn- field-was-fingerprinted?! [fingerprint-versions field-properties]