Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Apply FieldUserSettings on read instead of writing them into metabase_field (#82246)

Alexander Polyankin82cc16d2b54a
diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn
index a0f08d5c1e5..e65f1a9e443 100644
--- a/.clj-kondo/config.edn
+++ b/.clj-kondo/config.edn
@@ -73,6 +73,7 @@
:metabase/no-jsqlparser-imports {:level :warning}
:metabase/prefer-with-dynamic-fn-redefs {:level :warning}
:metabase/t2-query-namespace {:level :warning}
+ :metabase/table-or-field-query {:level :warning}
:metabase/toucan-model-ns {:level :warning}
:metabase/require-shape-checker {:level :warning}
:metabase/tests-must-live-in-known-modules {:level :warning}
@@ -900,7 +901,12 @@
:name app-db-namespaces}]
:config-in-ns
- {test-namespaces
+ {metabase.app-db.custom-migrations
+ {:linters
+ ;; a migration reads the rows as they were when it ran; applying today's user settings would make it drift
+ {:metabase/table-or-field-query {:level :off}}}
+
+ test-namespaces
{:linters
{:inline-def {:level :off}
:missing-docstring {:level :off}
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 178341a7dd9..43dbd0f726a 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -114,6 +114,7 @@
settings
users
util
+ warehouse-schema-overlay
warehouses}
:model-exports #{:model/Action}
:model-imports #{:model/Card
@@ -156,7 +157,8 @@
models
permissions
util
- util.config}
+ util.config
+ warehouse-schema-overlay}
:model-imports #{:model/Card
:model/Collection
:model/Dashboard
@@ -199,7 +201,8 @@
system
task
util
- util.config}
+ util.config
+ warehouse-schema-overlay}
:model-imports #{:model/CacheConfig
:model/Card
:model/Collection
@@ -209,7 +212,6 @@
:model/Database
:model/DatabaseRouter
:model/Document
- :model/Field
:model/ModerationReview
:model/PermissionsGroup
:model/Pulse
@@ -378,7 +380,8 @@
task
task.history
tracing
- util}
+ util
+ warehouse-schema-overlay}
:model-exports #{:model/AuditLog}
:model-imports #{:model/ApiKey
:model/Card
@@ -599,7 +602,8 @@
transforms
upload
util
- util.config}
+ util.config
+ warehouse-schema-overlay}
:model-exports #{:model/Collection}
:model-imports #{:model/Card
:model/CollectionBookmark
@@ -613,6 +617,7 @@
:model/Permissions
:model/Pulse
:model/Table
+ :model/TableUserSettings
:model/Timeline
:model/Transform
:model/User}}
@@ -632,7 +637,8 @@
premium-features
queries
request
- util}
+ util
+ warehouse-schema-overlay}
:model-imports #{:model/Card :model/Collection}}
comments
@@ -820,6 +826,7 @@
request
sync
util
+ warehouse-schema-overlay
warehouses.schema}
:model-imports #{:model/Database
:model/Dimension
@@ -857,7 +864,8 @@
query-processor
revisions
search
- util}
+ util
+ warehouse-schema-overlay}
:model-exports #{:model/Document}
:model-imports #{:model/Card
:model/Collection
@@ -910,6 +918,7 @@
util
util.classloader
util.config
+ warehouse-schema-overlay
warehouses
warehouses.schema}
:model-imports #{:model/Database :model/Field :model/Table}}
@@ -941,7 +950,8 @@
metabot
permissions
premium-features
- util.config}
+ util.config
+ warehouse-schema-overlay}
:model-imports #{:model/Card
:model/Collection
:model/ConnectionImpersonation
@@ -1170,7 +1180,8 @@
lib.schema
models
settings
- util}
+ util
+ warehouse-schema-overlay}
:model-imports #{:model/Card
:model/Database
:model/Dimension
@@ -1219,7 +1230,8 @@
lib.schema
lib.types
settings
- util}
+ util
+ warehouse-schema-overlay}
:model-imports #{:model/Card :model/Field :model/Table}}
lib.schema
@@ -1335,6 +1347,7 @@
users
util
util.config
+ warehouse-schema-overlay
warehouses.schema}
:model-imports #{:model/Card
:model/Collection
@@ -1387,7 +1400,8 @@
permissions
remote-sync
search
- util}
+ util
+ warehouse-schema-overlay}
:model-exports #{:model/Measure}
:model-imports #{:model/Collection :model/Table :model/User}}
@@ -1458,6 +1472,7 @@
transforms.base
util
util.config
+ warehouse-schema-overlay
warehouses
warehouses.schema}
:model-exports #{:model/AiUsageLog
@@ -1548,6 +1563,7 @@
sync
util
util.config
+ warehouse-schema-overlay
warehouses.schema}
:model-imports #{:model/Card
:model/Database
@@ -1602,7 +1618,8 @@
query-processor
request
server
- util}
+ util
+ warehouse-schema-overlay}
:model-imports #{:model/Card :model/Field :model/Measure}}
mfa
@@ -1641,7 +1658,6 @@
models
{:team "DevEx"
:api #{metabase.models.db
- metabase.models.humanization
metabase.models.init
metabase.models.interface
metabase.models.resolution
@@ -1656,9 +1672,9 @@
lib.legacy-mbql
lib.metadata
lib.schema
- settings
util
- util.classloader}
+ util.classloader
+ warehouse-schema-overlay}
:model-imports #{:model/Card
:model/Collection
:model/Dashboard
@@ -1788,6 +1804,7 @@
query-processor
util
util.classloader
+ warehouse-schema-overlay
warehouses.schema}
:model-imports #{:model/Card
:model/Dashboard
@@ -1816,6 +1833,7 @@
users
util
util.config
+ warehouse-schema-overlay
enterprise/permissions}
:model-exports #{:model/ApplicationPermissionsRevision
:model/CollectionPermissionGraphRevision
@@ -1842,7 +1860,8 @@
premium-features
request
settings
- util}
+ util
+ warehouse-schema-overlay}
:model-imports #{:model/Card
:model/Dashboard
:model/DataPermissions
@@ -2022,6 +2041,7 @@
util
util.config
util.graph
+ warehouse-schema-overlay
warehouses.schema}
:model-exports #{:model/Card
:model/ParameterCard
@@ -2066,7 +2086,8 @@
request
revisions
search
- util}
+ util
+ warehouse-schema-overlay}
:model-imports #{:model/Card
:model/CardBookmark
:model/Collection
@@ -2089,7 +2110,8 @@
permissions
query-processor
request
- util}
+ util
+ warehouse-schema-overlay}
:model-imports #{:model/Card :model/Field :model/Table}}
query-processor
@@ -2165,7 +2187,8 @@
tracing
users
util
- util.config}
+ util.config
+ warehouse-schema-overlay}
:model-imports #{:model/Card
:model/DashboardCardSeries
:model/Database
@@ -2341,7 +2364,8 @@
tracing
transforms
util
- util.config}
+ util.config
+ warehouse-schema-overlay}
:model-imports #{:model/Card
:model/Collection
:model/Dashboard
@@ -2381,6 +2405,7 @@
remote-sync
search
util
+ warehouse-schema-overlay
xrays}
:model-exports #{:model/Segment}
:model-imports #{:model/Collection :model/Table :model/User}}
@@ -2526,7 +2551,8 @@
lib.schema
settings
sql-tools.parsing
- util}
+ util
+ warehouse-schema-overlay}
:model-imports #{:model/Table}}
sql-tools.parsing
@@ -2622,6 +2648,7 @@
tracing
util
util.config
+ warehouse-schema-overlay
warehouses
warehouses.schema}
:model-imports #{:model/Database
@@ -2789,7 +2816,8 @@
tracing
transforms.base
transforms.indexes
- util}
+ util
+ warehouse-schema-overlay}
:model-exports #{:model/Transform
:model/TransformDagRun
:model/TransformJob
@@ -2832,7 +2860,8 @@
query-processor
sync
transforms.indexes
- util}}
+ util
+ warehouse-schema-overlay}}
transforms.indexes
{:team "Gadget"
@@ -2896,7 +2925,8 @@
models
permissions
system
- util}
+ util
+ warehouse-schema-overlay}
:model-imports #{:model/Action
:model/Card
:model/Collection
@@ -2933,6 +2963,7 @@
settings
sync
util
+ warehouse-schema-overlay
warehouses.schema}
:model-imports #{:model/Card
:model/Database
@@ -2955,7 +2986,8 @@
models
settings
task
- util}}
+ util
+ warehouse-schema-overlay}}
user-key-value
{:team "UX West"
@@ -3061,6 +3093,15 @@
:api #{metabase.graph.core}
:uses #{util}}
+ warehouse-schema-overlay
+ {:team "UX West"
+ :api #{metabase.warehouse-schema-overlay.core}
+ :uses #{util}
+ :model-imports #{:model/Field
+ :model/FieldUserSettings
+ :model/Table
+ :model/TableUserSettings}}
+
warehouses
{:team "UX West"
:module-exports #{warehouses.schema
@@ -3082,6 +3123,7 @@
settings
sync
util
+ warehouse-schema-overlay
warehouses.secrets}
:model-exports #{:model/Database}
:model-imports #{:model/Card
@@ -3116,6 +3158,7 @@
util
util.classloader
util.config
+ warehouse-schema-overlay
warehouses
warehouses.schema
warehouses.secrets
@@ -3129,14 +3172,18 @@
warehouses.schema
{:team "UX West"
:ns-prefix "metabase.warehouse-schema"
- :api #{metabase.warehouse-schema.field
+ :api #{metabase.warehouse-schema.core
+ metabase.warehouse-schema.field
metabase.warehouse-schema.field-values.distinct-batch
+ metabase.warehouse-schema.humanization
+ metabase.warehouse-schema.init
metabase.warehouse-schema.metadata-from-qp
metabase.warehouse-schema.metadata-queries
metabase.warehouse-schema.models.field
metabase.warehouse-schema.models.field-user-settings
metabase.warehouse-schema.models.field-values
metabase.warehouse-schema.models.table
+ metabase.warehouse-schema.models.table-user-settings
metabase.warehouse-schema.schema
metabase.warehouse-schema.table}
:uses #{api
@@ -3155,17 +3202,20 @@
query-processor
remote-sync
search
+ settings
sync.analyze
util
util.classloader
util.config
+ warehouse-schema-overlay
warehouses
enterprise/permissions}
:model-exports #{:model/Dimension
:model/Field
:model/FieldUserSettings
:model/FieldValues
- :model/Table}
+ :model/Table
+ :model/TableUserSettings}
:model-imports #{:model/Card
:model/Collection
:model/Database
@@ -3198,6 +3248,7 @@
sync
upload
util
+ warehouse-schema-overlay
warehouses.schema
xrays}
:model-imports #{:model/Collection
@@ -3243,6 +3294,7 @@
settings
sync.analyze
util
+ warehouse-schema-overlay
warehouses.schema}
:model-imports #{:model/Card
:model/Collection
@@ -3272,7 +3324,8 @@
models
query-processor
sync
- util}
+ util
+ warehouse-schema-overlay}
:model-imports #{:model/Database
:model/Field
:model/FieldValues
@@ -3358,6 +3411,7 @@
util
util.classloader
util.config
+ warehouse-schema-overlay
warehouses
warehouses.schema
enterprise/serialization}
@@ -3529,7 +3583,8 @@
premium-features
request
settings
- util}
+ util
+ warehouse-schema-overlay}
:model-imports #{:model/Card
:model/Collection
:model/DataPermissions
@@ -3549,11 +3604,12 @@
models
permissions
premium-features
- util}
+ util
+ warehouse-schema-overlay
+ warehouses.schema}
:model-imports #{:model/AuditLog
:model/Collection
:model/Dimension
- :model/Field
:model/Table
:model/User}}
@@ -3607,12 +3663,13 @@
transforms
transforms.base
util
- util.graph}
+ util.graph
+ warehouse-schema-overlay}
:model-imports #{:model/Card
:model/Collection
:model/Dashboard
:model/Document
- :model/Field
+ :model/FieldUserSettings
:model/Measure
:model/NativeQuerySnippet
:model/Sandbox
@@ -3642,6 +3699,7 @@
task
transforms
util
+ warehouse-schema-overlay
warehouses.schema
enterprise/dependencies}
:model-imports #{:model/Card
@@ -3672,6 +3730,7 @@
:uses #{lib.schema
premium-features
util
+ warehouse-schema-overlay
enterprise/sso}
:model-imports #{:model/Card
:model/Collection
@@ -3690,7 +3749,8 @@
collections
lib.schema
premium-features
- util}
+ util
+ warehouse-schema-overlay}
:model-imports #{:model/Card :model/Collection :model/Table}}
enterprise/mcp
@@ -3797,6 +3857,7 @@
task
util
util.config
+ warehouse-schema-overlay
enterprise/search.embeddings
enterprise/search.semantic}
:model-imports #{:model/Card
@@ -3854,6 +3915,7 @@
query-processor
remote-sync
util
+ warehouse-schema-overlay
warehouses
enterprise/permissions.impersonation}
:model-imports #{:model/ApplicationPermissionsRevision
@@ -3874,7 +3936,8 @@
models
permissions
query-processor
- util}
+ util
+ warehouse-schema-overlay}
:model-imports #{:model/Card :model/User}}
enterprise/permissions.impersonation
@@ -3918,6 +3981,7 @@
users
users.tenants
util
+ warehouse-schema-overlay
warehouses
warehouses.schema
enterprise/api}
@@ -3960,6 +4024,7 @@
task
task.history
util
+ warehouse-schema-overlay
enterprise/data-apps
enterprise/serialization
enterprise/transforms.python}
@@ -3975,6 +4040,7 @@
:model/PythonLibrary
:model/Segment
:model/Table
+ :model/TableUserSettings
:model/Timeline
:model/Transform
:model/TransformTag
@@ -4015,6 +4081,7 @@
search.entity-retrieval
task
util
+ warehouse-schema-overlay
enterprise/search.semantic}
:model-imports #{:model/Card
:model/Measure
@@ -4055,7 +4122,8 @@
task
tracing
util
- util.config}
+ util.config
+ warehouse-schema-overlay}
:model-imports #{:model/Collection :model/Dashboard :model/Table}}
enterprise/security-center
@@ -4239,7 +4307,8 @@
tracing
transforms
transforms.base
- util}
+ util
+ warehouse-schema-overlay}
:model-imports #{:model/Database
:model/Field
:model/Table
@@ -4268,7 +4337,8 @@
transforms
transforms.base
util
- util.config}
+ util.config
+ warehouse-schema-overlay}
:model-exports #{:model/PythonLibrary}
:model-imports #{:model/Database
:model/Field
@@ -4285,7 +4355,8 @@
permissions
premium-features
upload
- util}
+ util
+ warehouse-schema-overlay}
:model-imports #{:model/Database :model/Table}}
enterprise/warehouses
@@ -4297,6 +4368,7 @@
models
permissions
util
+ warehouse-schema-overlay
warehouses.schema}
:model-imports #{:model/Database :model/Field :model/Table}}}
diff --git a/.clj-kondo/ratchets.edn b/.clj-kondo/ratchets.edn
index e6de6a370ab..6de8521a587 100644
--- a/.clj-kondo/ratchets.edn
+++ b/.clj-kondo/ratchets.edn
@@ -62,6 +62,7 @@
:metabase/defsetting-namespace 1
:metabase/prefer-with-dynamic-fn-redefs 3
:metabase/t2-query-namespace 2
+ :metabase/table-or-field-query 1
:missing-docstring 1
:private-call 1
:redundant-ignore 1
diff --git a/.clj-kondo/src/hooks/metabase/toucan/db_ns.clj b/.clj-kondo/src/hooks/metabase/toucan/db_ns.clj
index cdfe8d20099..660d2d75c37 100644
--- a/.clj-kondo/src/hooks/metabase/toucan/db_ns.clj
+++ b/.clj-kondo/src/hooks/metabase/toucan/db_ns.clj
@@ -42,4 +42,5 @@
:message (format "Application database query calls like `%s` must live in metabase[-enterprise].<module>.db (or metabase.driver.<driver>.db) namespaces"
(hooks/sexpr fn-node))
:type :metabase/t2-query-namespace))))
+ ((requiring-resolve 'hooks.metabase.warehouse-schema-overlay.table-or-field-query/lint-read) input)
input)
diff --git a/.clj-kondo/src/hooks/metabase/warehouse_schema_overlay/table_or_field_query.clj b/.clj-kondo/src/hooks/metabase/warehouse_schema_overlay/table_or_field_query.clj
new file mode 100644
index 00000000000..1aadebe573e
--- /dev/null
+++ b/.clj-kondo/src/hooks/metabase/warehouse_schema_overlay/table_or_field_query.clj
@@ -0,0 +1,52 @@
+(ns hooks.metabase.warehouse-schema-overlay.table-or-field-query
+ "Lint that a query over `:model/Field` or `:model/Table` names its source with `field-query`/`table-query` from
+ `metabase.warehouse-schema-overlay.core`, or reads sync's values with `{:user-settings? false}`."
+ (:require
+ [clj-kondo.hooks-api :as hooks]))
+
+(def ^:private read-fn?
+ "The Toucan 2 query functions that read rows. Writes are not affected: they go to `metabase_field` itself."
+ #{"select" "select-one" "select-fn->fn" "select-fn->pk" "select-fn-reducible" "select-fn-set" "select-fn-vec"
+ "select-one-fn" "select-one-pk" "select-pk->fn" "select-pks-set" "select-pks-vec" "select-reducible"
+ "reducible-select" "reducible-query" "query" "query-one" "count" "exists?"})
+
+(def ^:private model->helper
+ {":model/Field" "field-query"
+ ":model/Table" "table-query"})
+
+(defn- test-file? [filename]
+ (boolean (and filename (re-find #"(?:^|/)test/" filename))))
+
+(defn- tokens
+ "Every token in `node`'s tree, as strings."
+ [node]
+ (tree-seq :children :children node))
+
+(defn- mentions?
+ [node pred]
+ (boolean (some (fn [n] (when-let [s (some-> (hooks/sexpr n) pr-str)] (pred s)))
+ (filter (complement :children) (tokens node)))))
+
+(defn- writes?
+ "Whether a `t2/query` form is really a write spelled as a query."
+ [node]
+ (mentions? node #{":update" ":delete-from" ":insert-into"}))
+
+(defn lint-read
+ "Register a `:metabase/table-or-field-query` finding when a read of `:model/Field` or `:model/Table` does not name
+ its source with the matching helper."
+ [{:keys [node filename]}]
+ (let [fn-node (first (:children node))
+ fn-name (some-> (hooks/sexpr fn-node) name)]
+ (when (and fn-name
+ (read-fn? fn-name)
+ (not (test-file? filename))
+ (not (writes? node)))
+ (doseq [[model helper] model->helper
+ :when (mentions? node #(= % model))
+ :when (not (mentions? node #(re-find (re-pattern (str "\\b" helper "$")) %)))]
+ (hooks/reg-finding!
+ (assoc (meta fn-node)
+ :message (format "A read of %s must name its source: add {:from [(%s)]} (or {:user-settings? false} for sync's own values)"
+ model helper)
+ :type :metabase/table-or-field-query))))))
diff --git a/dev/test/metabase/core/table_or_field_raw_usage_test.clj b/dev/test/metabase/core/table_or_field_raw_usage_test.clj
new file mode 100644
index 00000000000..21d8ca6a02e
--- /dev/null
+++ b/dev/test/metabase/core/table_or_field_raw_usage_test.clj
@@ -0,0 +1,188 @@
+(ns metabase.core.table-or-field-raw-usage-test
+ "Guards against hand-written Honey SQL that names `metabase_table`/`metabase_field` directly.
+
+ `metabase.warehouse-schema-overlay.core/table-query` and `/field-query` are the source a query over
+ `:model/Table`/`:model/Field` must read from, so that a user's Field values (`metabase_field_user_settings`) and a
+ workspace's table remapping (`workspace_table_remapping`) are applied. The `:metabase/table-or-field-query` clj-kondo
+ linter (`.clj-kondo/src/hooks/metabase/warehouse_schema_overlay/table_or_field_query.clj`) already catches this for
+ `t2/select`-style calls naming `:model/Table`/`:model/Field`, but it only fires on those Toucan 2 query functions.
+ Hand-written Honey SQL that instead names the raw table with the literal keyword `:metabase_table`/`:metabase_field`
+ -- in a `:from` or a join -- sails right past it; clj-kondo has no `:discouraged-keyword` linter to catch a bare
+ keyword literal irrespective of the function it's passed to (verified against the pinned 2026.07.24 build: only
+ `:discouraged-var` and `:discouraged-namespace` exist, both keyed on a resolved var/namespace symbol, not a keyword
+ value), and a hook can't fire on a keyword sitting in arbitrary data with no distinguishing enclosing call. Hence
+ this repo-level test instead, in the style of [[metabase.core.modules-test]]."
+ (:require
+ [clojure.java.io :as io]
+ [clojure.string :as str]
+ [clojure.test :refer :all]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private scanned-roots
+ "Where hand-written application-database Honey SQL lives. Driver modules are globbed since each has its own
+ `src` directory. `.clj-kondo/` (which spells out both keywords in the hook this test backs up) is simply never
+ one of these roots, so it needs no explicit exclusion."
+ (into ["src" "enterprise/backend/src"]
+ (comp (filter #(.isDirectory ^java.io.File %))
+ (map #(str "modules/drivers/" (.getName ^java.io.File %) "/src"))
+ (filter #(.isDirectory (io/file %))))
+ (some-> (io/file "modules/drivers") .listFiles)))
+
+(def ^:private excluded-paths
+ "Whole files/directories the scan never reports on, because a hit there isn't the linter's business:
+
+ - `metabase.warehouse-schema-overlay.core` itself builds `table-query`/`field-query` by naming the raw tables --
+ that's the one place allowed to.
+ - `metabase.sync.*` is sync's own read/write path: sync is the thing that writes `metabase_table`/`metabase_field`,
+ so it always wants the raw columns, and a separate change owns this namespace right now.
+ - Liquibase/custom migrations run against whatever shape the app DB had at that migration's version, before (or
+ long after) `field-query`/`table-query` existed; they are not reads a *current* user sees rendered anywhere."
+ #{"src/metabase/warehouse_schema_overlay/core.clj"})
+
+(defn- excluded-dir?
+ [path]
+ (or (str/starts-with? path "src/metabase/sync/")
+ (str/includes? path "custom_migrations")))
+
+(def ^:private allowed-raw-usages
+ "Files where a `:metabase_table`/`:metabase_field` literal survives review: every hit in the file was read back and
+ confirmed to select only columns `table-query`/`field-query` don't touch (an id, a count, a foreign key used purely
+ to join or filter), or the file has its own structural reason to want the raw row. Note that a Table's
+ `display_name`, `description`, `is_published` and `collection_id` are user values like a Field's -- a rationale
+ resting on those being sync-owned no longer holds. Add an entry here only after
+ checking every hit it covers -- see the check's failure message for the two helpers this is standing in for.
+
+ Keep this in sync with what the scan actually finds: [[stale-allowed-raw-usage-test]] fails on an entry that no
+ longer matches anything, same as clj-kondo's ignore ratchets flag a stale exemption."
+ {"src/metabase/usage_metadata/db.clj"
+ "raw-field-fingerprint selects only :fingerprint, a sync-owned column with no user override."
+
+ "src/metabase/collections/children.clj"
+ "the :model case joins metabase_table only for :is_upload, which is sync-owned."
+
+ "src/metabase/queries/db.clj"
+ "field-database-info-for-ids reads the Field's own raw :name (sync-owned, not user-settable); its Table join is
+ already routed through table-query."
+
+ "src/metabase/models/db.clj"
+ "field-in-path-query is a nested id-only lookup used as a :where filter, not a value read."
+
+ "src/metabase/warehouses_rest/db.clj"
+ "delete-field-values-for-database! selects only :f.id inside a DELETE's subquery; database-usage-counts only
+ counts rows. autocomplete-fields' Table join now goes through table-query."
+
+ "src/metabase/parameters/db.clj"
+ "both joins bring in the Table only to filter (.active, .db_id) or project its :id; neither selects
+ :schema/:name."
+
+ "enterprise/backend/src/metabase_enterprise/dependencies/db.clj"
+ "the remaining hits select only :id, to build permission and dependency filters. The two that read a Table's
+ display_name -- a user value -- now go through table-query."
+
+ "enterprise/backend/src/metabase_enterprise/remote_sync/db.clj"
+ "remote sync matches and tracks content by its actual physical (db, schema, table) location across instances --
+ overlaying the workspace-remapped name would work against the one thing this module does."
+
+ "enterprise/backend/src/metabase_enterprise/data_complexity_score/db.clj"
+ "active-field-counts-by-table only counts active Fields per table; no Field/Table content column is read."})
+
+(defn- source-file?
+ [^java.io.File file]
+ (and (.isFile file)
+ (let [name (.getName file)]
+ (or (str/ends-with? name ".clj") (str/ends-with? name ".cljc")))))
+
+(defn- source-files
+ []
+ (for [root scanned-roots
+ ^java.io.File file (file-seq (io/file root))
+ :when (source-file? file)]
+ (.getPath file)))
+
+(defn- strip-strings-and-comments
+ "Blank out string-literal and line-comment content in `text`, keeping newlines so line numbers still line up.
+ A naive keyword scan over raw source text would otherwise flag docstrings and comments that merely *mention*
+ `:metabase_table`/`:metabase_field` (see e.g. `metabase.warehouse-schema.schema`'s column-list docstrings)."
+ [^String text]
+ (let [n (.length text)
+ sb (StringBuilder. n)]
+ (loop [i 0, in-str? false]
+ (if (>= i n)
+ (str sb)
+ (let [c (.charAt text i)]
+ (cond
+ (and in-str? (= c \\) (< (inc i) n))
+ (do (.append sb (if (= (.charAt text (inc i)) \newline) "\n " " "))
+ (recur (+ i 2) true))
+
+ (= c \")
+ (do (.append sb \space)
+ (recur (inc i) (not in-str?)))
+
+ in-str?
+ (do (.append sb (if (= c \newline) \newline \space))
+ (recur (inc i) true))
+
+ (= c \;)
+ (let [nl (long (or (str/index-of text "\n" i) n))]
+ (.append sb (apply str (repeat (- nl i) \space)))
+ (recur nl false))
+
+ :else
+ (do (.append sb c)
+ (recur (inc i) false))))))))
+
+(def ^:private raw-usage-pattern
+ "A `:metabase_table`/`:metabase_field` keyword literal naming a table -- as opposed to `:metabase_field.column` or
+ `:metabase_field/column`, which qualify a column of a source named elsewhere (often `table-query`/`field-query`'s
+ own default alias, which is the raw table name)."
+ #":metabase_(?:table|field)(?![./\w])")
+
+(defn- raw-usage-hits
+ "Every `{:file :line :text}` where `path`'s code (not its strings or comments) uses the literal keyword to name a
+ table, skipping `t2/table-name` definitions -- the canonical place the keyword has to be spelled out."
+ [path]
+ (let [text (slurp path)
+ code-only (strip-strings-and-comments text)
+ lines (str/split-lines text)
+ code-lines (str/split-lines code-only)]
+ (into []
+ (keep (fn [[line-no orig code]]
+ (when (and (re-find raw-usage-pattern code)
+ (not (str/includes? orig "t2/table-name")))
+ {:file path, :line line-no, :text (str/trim orig)})))
+ (map vector (rest (range)) lines code-lines))))
+
+(defn- relative-path
+ [path]
+ (str/replace path #"^\./" ""))
+
+(defn- all-hits
+ []
+ (into []
+ (mapcat raw-usage-hits)
+ (remove excluded-dir? (map relative-path (source-files)))))
+
+(defn- excused?
+ [file]
+ (boolean (or (contains? excluded-paths file)
+ (contains? allowed-raw-usages file))))
+
+(deftest no-raw-metabase-table-or-field-honey-sql-test
+ (testing (str "Hand-written Honey SQL must not name `:metabase_table`/`:metabase_field` directly.\n"
+ "Read from `(metabase.warehouse-schema-overlay.core/table-query)` / `(field-query)` instead"
+ " (or add `{:alias ...}` for a join), or pass `{:user-settings? false}` for a deliberate raw read.\n"
+ "If the hit really is fine as-is (only an id, a count, or a foreign key used to join/filter), review"
+ " it and add it to `allowed-raw-usages` in this test with a one-line reason.")
+ (doseq [{:keys [file line text]} (all-hits)]
+ (testing (format "\n%s:%d\n %s" file line text)
+ (is (excused? file) "unexpected raw metabase_table/metabase_field usage (see message above)")))))
+
+(deftest stale-allowed-raw-usage-test
+ (testing "every allowed-raw-usages entry still matches at least one hit, so the allowlist doesn't accumulate cruft"
+ (let [files-with-hits (into #{} (map :file) (all-hits))]
+ (doseq [file (keys allowed-raw-usages)]
+ (testing (format "\n%s" file)
+ (is (contains? files-with-hits file)
+ "no raw metabase_table/metabase_field usage found here anymore -- remove this allowlist entry"))))))
diff --git a/enterprise/backend/src/metabase_enterprise/action_v2/db.clj b/enterprise/backend/src/metabase_enterprise/action_v2/db.clj
index 31a8a6567b7..23d16a01462 100644
--- a/enterprise/backend/src/metabase_enterprise/action_v2/db.clj
+++ b/enterprise/backend/src/metabase_enterprise/action_v2/db.clj
@@ -7,17 +7,18 @@
[metabase.util :as u]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
(mu/defn table
"The Table with `table-id`, or nil."
[table-id :- ::lib.schema.id/table]
- (t2/select-one :model/Table table-id))
+ (t2/select-one :model/Table :id table-id {:from [(warehouse-schema-overlay/table-query)]}))
(mu/defn active-table
"The active Table with `table-id`, or nil."
[table-id :- ::lib.schema.id/table]
- (t2/select-one :model/Table :id table-id :active true))
+ (t2/select-one :model/Table :id table-id :active true {:from [(warehouse-schema-overlay/table-query)]}))
(mu/defn database
"The Database with `database-id`, or nil."
@@ -27,28 +28,29 @@
(mu/defn fields
"The Fields with `field-ids`."
[field-ids :- [:sequential ::lib.schema.id/field]]
- (t2/select :model/Field :id [:in field-ids]))
+ (t2/select :model/Field :id [:in field-ids] {:from [(warehouse-schema-overlay/field-query)]}))
(mu/defn pk-fields-for-table
"The active primary key Fields of the Table with `table-id`."
[table-id :- ::lib.schema.id/table]
- (t2/select :model/Field :table_id table-id :semantic_type :type/PK :active true))
+ (t2/select :model/Field :table_id table-id :semantic_type :type/PK :active true {:from [(warehouse-schema-overlay/field-query)]}))
(mu/defn fields-by-name
"The Fields of the Table with `table-id` named one of `field-names`."
[table-id :- ::lib.schema.id/table
field-names :- [:sequential :string]]
- (t2/select :model/Field :table_id table-id :name [:in field-names]))
+ (t2/select :model/Field :table_id table-id :name [:in field-names] {:from [(warehouse-schema-overlay/field-query)]}))
(mu/defn active-fields-in-position-order
"The active Fields of the Table with `table-id`, in position order."
[table-id :- ::lib.schema.id/table]
- (t2/select :model/Field :table_id table-id :active true {:order-by [[:position]]}))
+ (t2/select :model/Field :table_id table-id :active true {:from [(warehouse-schema-overlay/field-query)]
+ :order-by [[:position]]}))
(mu/defn field-requirements-by-name
"A map of name to the name, required flag, and base type of the Fields of the Table with `table-id`."
[table-id :- ::lib.schema.id/table]
- (t2/select-fn->fn :name identity [:model/Field :name :database_required :base_type] :table_id table-id))
+ (t2/select-fn->fn :name identity [:model/Field :name :database_required :base_type] :table_id table-id {:from [(warehouse-schema-overlay/field-query)]}))
(mu/defn category-list-field-ids-by-name
"The `:id` and `:lower_name` rows of the category list Fields of the Table with `table-id` whose name matches one
@@ -56,7 +58,7 @@
[table-id :- ::lib.schema.id/table
names :- [:sequential :string]]
(t2/query {:select [:id [[:lower :name] :lower_name]]
- :from [(t2/table-name :model/Field)]
+ :from [(warehouse-schema-overlay/field-query)]
:where [:and
[:= :table_id table-id]
[:in [:lower :name] (map u/lower-case-en names)]
diff --git a/enterprise/backend/src/metabase_enterprise/advanced_permissions/db.clj b/enterprise/backend/src/metabase_enterprise/advanced_permissions/db.clj
index cd6ca6d57bc..61b444a6121 100644
--- a/enterprise/backend/src/metabase_enterprise/advanced_permissions/db.clj
+++ b/enterprise/backend/src/metabase_enterprise/advanced_permissions/db.clj
@@ -6,12 +6,13 @@
[metabase.util.honey-sql-2 :as h2x]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
(mu/defn table
"The Table with `table-id`, or nil."
[table-id :- ::lib.schema.id/table]
- (t2/select-one :model/Table :id table-id))
+ (t2/select-one :model/Table :id table-id {:from [(warehouse-schema-overlay/table-query)]}))
(mu/defn group-manager?
"Whether the User with `user-id` manages the PermissionsGroup with `group-id`."
@@ -42,7 +43,8 @@
database-ids :- [:sequential ::lib.schema.id/database]]
(t2/query {:select [[:t.db_id :db_id]]
:from [[(t2/table-name :model/Sandbox) :s]]
- :join [[(t2/table-name :model/Table) :t] [:= :s.table_id :t.id]]
+ :join [(warehouse-schema-overlay/table-query {:alias :t, :user-settings? false})
+ [:= :s.table_id :t.id]]
:where [:and
[:= :s.group_id group-id]
[:in :t.db_id database-ids]]}))
@@ -83,7 +85,8 @@
group-ids :- [:sequential ms/PositiveInt]]
(t2/query {:select [[:s.group_id :group_id]]
:from [[(t2/table-name :model/Sandbox) :s]]
- :join [[(t2/table-name :model/Table) :t] [:= :t.id :s.table_id]]
+ :join [(warehouse-schema-overlay/table-query {:alias :t, :user-settings? false})
+ [:= :t.id :s.table_id]]
:where [:and
[:in :s.group_id group-ids]
[:= :t.db_id database-id]]}))
diff --git a/enterprise/backend/src/metabase_enterprise/audit_app/db.clj b/enterprise/backend/src/metabase_enterprise/audit_app/db.clj
index 17b16d6d9a7..49c9b2fabcc 100644
--- a/enterprise/backend/src/metabase_enterprise/audit_app/db.clj
+++ b/enterprise/backend/src/metabase_enterprise/audit_app/db.clj
@@ -6,6 +6,7 @@
[metabase.lib.schema.id :as lib.schema.id]
[metabase.queries.schema :as queries.schema]
[metabase.util.malli :as mu]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[metabase.warehouse-schema.schema :as warehouse-schema.schema]
[metabase.warehouses.schema :as warehouses.schema]
[toucan2.core :as t2]))
@@ -56,17 +57,18 @@
"The Tables of the Database with `database-id` named one of `table-names`."
[database-id :- ::lib.schema.id/database
table-names :- [:or [:set :string] [:sequential :string]]]
- (t2/select :model/Table :db_id database-id :name [:in table-names]))
+ (t2/select :model/Table :db_id database-id :name [:in table-names] {:from [(warehouse-schema-overlay/table-query)]}))
(mu/defn tables-of-database-in-id-order
"The `:id`, `:name`, `:schema`, and `:active` of the Tables of the Database with `database-id`, in ID order."
[database-id :- ::lib.schema.id/database]
- (t2/select [:model/Table :id :name :schema :active] :db_id database-id {:order-by [[:id :asc]]}))
+ (t2/select [:model/Table :id :name :schema :active] :db_id database-id {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]
+ :order-by [[:id :asc]]}))
(mu/defn active-public-table-exists?
"Whether the Database with `database-id` has an active Table in the `public` schema."
[database-id :- ::lib.schema.id/database]
- (t2/exists? :model/Table :db_id database-id :schema "public" :active true))
+ (t2/exists? :model/Table :db_id database-id :schema "public" :active true {:from [(warehouse-schema-overlay/table-query)]}))
(mu/defn table-ids-to-downcase
"The IDs of the Metabase-managed Tables of the Database with `database-id` that have no lower-cased counterpart
@@ -74,14 +76,14 @@
[database-id :- ::lib.schema.id/database]
(mapv :id
(t2/query {:select [:table.id]
- :from [[(t2/table-name :model/Table) :table]]
+ :from [(warehouse-schema-overlay/table-query {:alias :table, :user-settings? false})]
:where [:and [:= :table.db_id database-id]
;; Exclude DATABASECHANGELOG, DATABASECHANGELOGLOCK, and QRTZ_* tables, they are not metabase managed
[:not= :table.name "DATABASECHANGELOG"]
[:not= :table.name "DATABASECHANGELOGLOCK"] ;; new instances do not get this file, but existing instances may have it
[:not [:like :table.name "QRTZ_%"]]
[:not [:exists ^:allow-subquery {:select [1]
- :from [[(t2/table-name :model/Table) :self_table]]
+ :from [(warehouse-schema-overlay/table-query {:alias :self_table, :user-settings? false})]
:where [:and
[:= :self_table.db_id :table.db_id]
[:or
@@ -102,15 +104,15 @@
[database-id :- ::lib.schema.id/database]
(mapv :id
(t2/query {:select [:field.id]
- :from [[(t2/table-name :model/Field) :field]]
- :inner-join [[(t2/table-name :model/Table) :table]
+ :from [(warehouse-schema-overlay/field-query {:alias :field})]
+ :inner-join [(warehouse-schema-overlay/table-query {:alias :table})
[:= :table.id :field.table_id]]
:where [:and [:= :table.db_id database-id]
[:not= :table.name "DATABASECHANGELOG"]
[:not [:like :table.name "QRTZ_%"]]
[:not [:exists ^:allow-subquery {:select [1]
- :from [[(t2/table-name :model/Field) :self_field]]
- :inner-join [[(t2/table-name :model/Table) :self_table]
+ :from [(warehouse-schema-overlay/field-query {:alias :self_field})]
+ :inner-join [(warehouse-schema-overlay/table-query {:alias :self_table})
[:= :self_table.id :self_field.table_id]]
:where [:and
[:= :self_table.db_id :table.db_id]
@@ -161,7 +163,7 @@
(mu/defn field-names-of-table
"The `:id` and `:name` of the Fields of the Table with `table-id`."
[table-id :- ::lib.schema.id/table]
- (t2/select [:model/Field :id :name] :table_id table-id))
+ (t2/select [:model/Field :id :name] :table_id table-id {:from [(warehouse-schema-overlay/field-query {:user-settings? false})]}))
(mu/defn card-result-metadata-reducible
"Reducible raw `:id` and `:result_metadata` rows of the Cards of the Database with `database-id`."
diff --git a/enterprise/backend/src/metabase_enterprise/audit_app/pages/queries.clj b/enterprise/backend/src/metabase_enterprise/audit_app/pages/queries.clj
index b477b6c54a1..b511bf3449d 100644
--- a/enterprise/backend/src/metabase_enterprise/audit_app/pages/queries.clj
+++ b/enterprise/backend/src/metabase_enterprise/audit_app/pages/queries.clj
@@ -5,7 +5,8 @@
[metabase-enterprise.audit-app.pages.common :as common]
[metabase-enterprise.audit-app.pages.common.cards :as cards]
[metabase.app-db.core :as mdb]
- [metabase.audit-app.core :as audit]))
+ [metabase.audit-app.core :as audit]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]))
;; List of all failing questions
(defmethod audit.i/internal-query ::bad-table
@@ -75,7 +76,7 @@
:from [[:report_card :card]]
:left-join [[:collection :coll] [:= :card.collection_id :coll.id]
[:metabase_database :db] [:= :card.database_id :db.id]
- [:metabase_table :t] [:= :card.table_id :t.id]
+ (warehouse-schema-overlay/table-query {:alias :t}) [:= :card.table_id :t.id]
[:core_user :u] [:= :card.creator_id :u.id]
:latest_qe [:= :card.id :latest_qe.card_id]
:query_runs [:= :card.id :query_runs.card_id]
diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/db.clj b/enterprise/backend/src/metabase_enterprise/data_apps/db.clj
index ad5c7a8e19f..522c39d8e9c 100644
--- a/enterprise/backend/src/metabase_enterprise/data_apps/db.clj
+++ b/enterprise/backend/src/metabase_enterprise/data_apps/db.clj
@@ -6,6 +6,7 @@
[metabase.util :as u]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
(def ^:private non-blob-columns
@@ -52,13 +53,15 @@
(defn table-database-id
"The database ID for the Table with `table-id`, or nil."
[table-id]
- (t2/select-one-fn :db_id :model/Table :id table-id))
+ (t2/select-one-fn :db_id :model/Table :id table-id
+ {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))
(defn existing-table-ids
"The IDs from `table-ids` that belong to existing Tables."
[table-ids]
(if (seq table-ids)
- (t2/select-pks-set :model/Table :id [:in table-ids])
+ (t2/select-pks-set :model/Table :id [:in table-ids]
+ {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]})
#{}))
(defn users-for-permission-warnings
@@ -196,7 +199,7 @@
:t.schema
[:t.db_id :database_id]
[:d.name :database_name]]
- :from [[:metabase_table :t]]
+ :from [(warehouse-schema-overlay/table-query {:alias :t})]
:join [[:metabase_database :d] [:= :d.id :t.db_id]]
:where [:in :t.id table-ids]
:order-by [[:d.name :asc] [:t.schema :asc] [:t.display_name :asc]]}))
@@ -222,11 +225,12 @@
:from [[:permissions_group_membership :pgm]]
:join [[:data_permissions :dp] [:= :dp.group_id :pgm.group_id]
[:permissions_group :pg] [:= :pg.id :pgm.group_id]
- [:metabase_table :t] [:and
- [:= :t.db_id :dp.db_id]
- [:or
- [:= :dp.table_id nil]
- [:= :dp.table_id :t.id]]]]
+ (warehouse-schema-overlay/table-query {:alias :t, :user-settings? false})
+ [:and
+ [:= :t.db_id :dp.db_id]
+ [:or
+ [:= :dp.table_id nil]
+ [:= :dp.table_id :t.id]]]]
:where [:and
[:in :pgm.user_id user-ids]
[:in :t.id table-ids]
diff --git a/enterprise/backend/src/metabase_enterprise/data_complexity_score/db.clj b/enterprise/backend/src/metabase_enterprise/data_complexity_score/db.clj
index 845940c2782..74d2a67560c 100644
--- a/enterprise/backend/src/metabase_enterprise/data_complexity_score/db.clj
+++ b/enterprise/backend/src/metabase_enterprise/data_complexity_score/db.clj
@@ -10,6 +10,7 @@
[metabase.util.honey-sql-2 :as h2x]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
(mu/defn active-field-counts-by-table
@@ -64,7 +65,7 @@
[audit-database-id :- ::lib.schema.id/database]
(t2/select [:model/Table :id :name :collection_id :is_published :visibility_type :db_id :data_layer :data_authority]
:active true
- :db_id [:not= audit-database-id]))
+ :db_id [:not= audit-database-id] {:from [(warehouse-schema-overlay/table-query)]}))
(mu/defn metabot-by-entity-id
"The Metabot with `entity-id`, or nil."
diff --git a/enterprise/backend/src/metabase_enterprise/data_studio/db.clj b/enterprise/backend/src/metabase_enterprise/data_studio/db.clj
index ba8d8ea1a7d..93d7f9829ab 100644
--- a/enterprise/backend/src/metabase_enterprise/data_studio/db.clj
+++ b/enterprise/backend/src/metabase_enterprise/data_studio/db.clj
@@ -7,6 +7,8 @@
[metabase.lib.schema.id :as lib.schema.id]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
+ [metabase.warehouse-schema.models.table-user-settings :as schema.table-user-settings]
[toucan2.core :as t2]))
(def ^:private TableSelectors
@@ -50,9 +52,9 @@
[:not [:in output-table-id tables]])]
(t2/reducible-query {:select [[output-table-id :table_id]]
:from [[(t2/table-name :model/Dimension) :dim]]
- :join [[(t2/table-name :model/Field) :source_field]
+ :join [(warehouse-schema-overlay/field-query {:alias :source_field, :user-settings? false})
[:= :dim.field_id :source_field.id]
- [(t2/table-name :model/Field) :target_field]
+ (warehouse-schema-overlay/field-query {:alias :target_field, :user-settings? false})
[:= :dim.human_readable_field_id :target_field.id]]
:where [:and
[:= :dim.type "external"]
@@ -66,7 +68,8 @@
extra-table-ids :- [:maybe [:set ::lib.schema.id/table]]
extra-mode :- [:enum :unpublished :any]]
(t2/select-pks-set :model/Table
- {:where (let [selector-expr (table-selectors-expr selectors)]
+ {:from [(warehouse-schema-overlay/table-query)]
+ :where (let [selector-expr (table-selectors-expr selectors)]
(if (seq extra-table-ids)
[:or selector-expr (case extra-mode
:unpublished [:and [:in :id extra-table-ids] [:= :is_published false]]
@@ -76,12 +79,12 @@
(mu/defn published-table-ids
"The IDs of the published Tables among `table-ids`."
[table-ids :- [:set ::lib.schema.id/table]]
- (t2/select-pks-set :model/Table :id [:in table-ids] :is_published true))
+ (t2/select-pks-set :model/Table :id [:in table-ids] :is_published true {:from [(warehouse-schema-overlay/table-query)]}))
(mu/defn tables
"The Tables with `table-ids`."
[table-ids :- [:set ::lib.schema.id/table]]
- (t2/select :model/Table :id [:in table-ids]))
+ (t2/select :model/Table :id [:in table-ids] {:from [(warehouse-schema-overlay/table-query)]}))
(mu/defn collection
"The Collection with `collection-id`, or nil."
@@ -106,12 +109,14 @@
"Publish the Tables with `table-ids` into the Collection with `collection-id`, returning the number updated."
[table-ids :- [:set ::lib.schema.id/table]
collection-id :- ::lib.schema.id/collection]
- (t2/update! :model/Table :id [:in table-ids] {:collection_id collection-id, :is_published true}))
+ (schema.table-user-settings/upsert-user-settings-for-tables!
+ table-ids {:collection_id collection-id, :is_published true}))
(mu/defn unpublish-tables!
"Unpublish the Tables with `table-ids` and detach them from their Collection, returning the number updated."
[table-ids :- [:set ::lib.schema.id/table]]
- (t2/update! :model/Table :id [:in table-ids] {:collection_id nil, :is_published false}))
+ (schema.table-user-settings/upsert-user-settings-for-tables!
+ table-ids {:collection_id nil, :is_published false}))
(mu/defn published-table-visible-to-user?
"Whether the Table with `table-id` is published in a Collection the User with `user-id` can read."
@@ -119,7 +124,8 @@
user-id :- ::lib.schema.id/user
superuser? :- :boolean]
(t2/exists? :model/Table
- {:where [:and
+ {:from [(warehouse-schema-overlay/table-query)]
+ :where [:and
[:= :id table-id]
[:= :is_published true]
(collection/visible-collection-filter-clause
@@ -130,7 +136,8 @@
"Whether the current user can read the Collection of any published Table."
[]
(t2/exists? :model/Table
- {:where [:and
+ {:from [(warehouse-schema-overlay/table-query)]
+ :where [:and
[:= :is_published true]
(collection/visible-collection-filter-clause :collection_id)]}))
@@ -138,7 +145,8 @@
"Whether the current user can read the Collection of any published Table in the Database with `database-id`."
[database-id :- ::lib.schema.id/database]
(t2/exists? :model/Table
- {:where [:and
+ {:from [(warehouse-schema-overlay/table-query)]
+ :where [:and
[:= :db_id database-id]
[:= :is_published true]
(collection/visible-collection-filter-clause :collection_id)]}))
diff --git a/enterprise/backend/src/metabase_enterprise/data_studio/permissions/published_tables.clj b/enterprise/backend/src/metabase_enterprise/data_studio/permissions/published_tables.clj
index 904d263291c..a08a3329b5c 100644
--- a/enterprise/backend/src/metabase_enterprise/data_studio/permissions/published_tables.clj
+++ b/enterprise/backend/src/metabase_enterprise/data_studio/permissions/published_tables.clj
@@ -7,7 +7,8 @@
[metabase.models.interface :as mi]
[metabase.permissions.core :as perms]
[metabase.premium-features.core :refer [defenterprise]]
- [metabase.util.honey-sql-2 :as h2x]))
+ [metabase.util.honey-sql-2 :as h2x]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]))
(defenterprise user-published-table-permission
"Returns :query-builder permission if table is published and user has collection access.
@@ -44,7 +45,7 @@
[:in table-id-column
^:allow-subquery
{:select [:id]
- :from [:metabase_table]
+ :from [(warehouse-schema-overlay/table-query)]
:where [:and
[:= :is_published true]
(collection/visible-collection-filter-clause
@@ -67,7 +68,7 @@
{:select [[:mt.id :id]
[(h2x/literal :perms/create-queries) :perm_type]
[(h2x/literal :query-builder) :perm_value]]
- :from [[:metabase_table :mt]]
+ :from [(warehouse-schema-overlay/table-query {:alias :mt})]
:where (cond-> [:and
[:= :mt.is_published true]
(collection/visible-collection-filter-clause
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/db.clj b/enterprise/backend/src/metabase_enterprise/dependencies/db.clj
index df6878452e4..8652b590536 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/db.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/db.clj
@@ -16,6 +16,7 @@
[metabase.util.honey-sql-2 :as h2x]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
(def ^:private VisibleOpts
@@ -277,7 +278,7 @@
nil))]
{:table-name (case entity-type
:card :report_card
- :table :metabase_table
+ :table (first (warehouse-schema-overlay/table-query))
:transform :transform
:snippet :native_query_snippet
:dashboard :report_dashboard
@@ -427,7 +428,7 @@
(:collection joins) (conj :collection [:= :entity.collection_id :collection.id])
(:dashboard joins) (conj [:report_dashboard :dashboard] [:= :entity.dashboard_id :dashboard.id])
(:document joins) (conj :document [:= :entity.document_id :document.id])
- (:table joins) (conj [:metabase_table :table] [:= :entity.table_id :table.id])))
+ (:table joins) (conj (warehouse-schema-overlay/table-query {:alias :table}) [:= :entity.table_id :table.id])))
(defn- dependency-item-select
"The per-entity-type SELECT that [[dependency-item-ids]] and [[dependency-item-count]] union together, matching
@@ -549,19 +550,28 @@
ids :- [:set ::deps.dependency-types/entity-id]]
(t2/select (deps.dependency-types/dependency-type->model entity-type) :id [:in ids]))
+(defn- entity-source
+ "What a read of `entity-type` selects from: the overlay for a Table, else the model's own table."
+ [entity-type]
+ (if (= entity-type :table)
+ {:from [(warehouse-schema-overlay/table-query)]}
+ {}))
+
(mu/defn instances-with-columns
"The `columns` of the instances of the entity type `entity-type` with `ids`."
[entity-type :- ::deps.dependency-types/dependency-types
columns :- [:sequential :keyword]
ids :- [:sequential ::deps.dependency-types/entity-id]]
- (t2/select (into [(deps.dependency-types/dependency-type->model entity-type)] columns) :id [:in ids]))
+ (t2/select (into [(deps.dependency-types/dependency-type->model entity-type)] columns)
+ :id [:in ids] (entity-source entity-type)))
(mu/defn instance-with-columns
"The `columns` of the instance of the entity type `entity-type` with `id`, or nil."
[entity-type :- ::deps.dependency-types/dependency-types
columns :- [:sequential :keyword]
id :- ::deps.dependency-types/entity-id]
- (t2/select-one (into [(deps.dependency-types/dependency-type->model entity-type)] columns) :id id))
+ (t2/select-one (into [(deps.dependency-types/dependency-type->model entity-type)] columns)
+ :id id (entity-source entity-type)))
(mu/defn card
"The Card with `card-id`, or nil."
@@ -587,19 +597,19 @@
(mu/defn tables
"The Tables with `table-ids`."
[table-ids :- [:set ::lib.schema.id/table]]
- (t2/select :model/Table :id [:in table-ids]))
+ (t2/select :model/Table :id [:in table-ids] {:from [(warehouse-schema-overlay/table-query)]}))
(mu/defn table-database-ids
"The `:id` and `:db_id` of the Tables with `table-ids`."
[table-ids :- [:sequential ::lib.schema.id/table]]
- (t2/select [:model/Table :id :db_id] :id [:in table-ids]))
+ (t2/select [:model/Table :id :db_id] :id [:in table-ids] {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))
(mu/defn table-id-by-name
"The ID of the Table named `table-name` in `schema` of the Database with `db-id`, or nil."
[db-id :- ::lib.schema.id/database
schema :- [:maybe :string]
table-name :- :string]
- (t2/select-one-fn :id :model/Table :db_id db-id :schema schema :name table-name))
+ (t2/select-one-fn :id :model/Table :db_id db-id :schema schema :name table-name {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))
(mu/defn transform-sources
"The `:id` and `:source` of the Transforms with `transform-ids`."
@@ -857,17 +867,24 @@
:limit batch-size})))
(mu/defn table-ids-with-outdated-findings
- "The IDs of the Tables of the Database with `db-id` whose dependents were analyzed before the Table or one of
- its Fields last changed."
+ "The IDs of the Tables of the Database with `db-id` whose dependents were analyzed before the Table, one of its
+ Fields, or one of its Fields' user settings last changed."
[db-id :- ::lib.schema.id/database]
(t2/select-fn-set :table_id :model/AnalysisFinding
{:select [:field_updates/table_id]
:from [[^:allow-subquery {:select [[:table/id :table_id]
[:table/updated_at :last_table_update]
- [[:max :field/updated_at] :last_field_update]]
- :from [[(t2/table-name :model/Table) :table]]
- :left-join [[(t2/table-name :model/Field) :field]
- [:= :field/table_id :table/id]]
+ [[:max [:case
+ [:>= :field/updated_at
+ [:coalesce :field_settings/updated_at :field/updated_at]]
+ :field/updated_at
+ :else :field_settings/updated_at]]
+ :last_field_update]]
+ :from [(warehouse-schema-overlay/table-query {:alias :table})]
+ :left-join [(warehouse-schema-overlay/field-query {:alias :field})
+ [:= :field/table_id :table/id]
+ [(t2/table-name :model/FieldUserSettings) :field_settings]
+ [:= :field_settings/field_id :field/id]]
:where [:= :table/db_id db-id]
:group-by [:table/id
:table/updated_at]}
diff --git a/enterprise/backend/src/metabase_enterprise/embedding_hub/db.clj b/enterprise/backend/src/metabase_enterprise/embedding_hub/db.clj
index 7fb65163889..5675c165989 100644
--- a/enterprise/backend/src/metabase_enterprise/embedding_hub/db.clj
+++ b/enterprise/backend/src/metabase_enterprise/embedding_hub/db.clj
@@ -4,6 +4,7 @@
(:require
[metabase.lib.schema.id :as lib.schema.id]
[metabase.util.malli :as mu]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
(mu/defn user-database-exists?
@@ -21,7 +22,8 @@
(mu/defn upload-table-exists-in-database?
"Whether the Database with `database-id` holds an active uploaded Table."
[database-id :- ::lib.schema.id/database]
- (t2/exists? :model/Table {:where [:and
+ (t2/exists? :model/Table {:from [(warehouse-schema-overlay/table-query)]
+ :where [:and
[:= :active true]
[:= :is_upload true]
[:= :db_id database-id]]}))
diff --git a/enterprise/backend/src/metabase_enterprise/entity_retrieval/db.clj b/enterprise/backend/src/metabase_enterprise/entity_retrieval/db.clj
index f1a7882b7a4..1c8dc2de4d3 100644
--- a/enterprise/backend/src/metabase_enterprise/entity_retrieval/db.clj
+++ b/enterprise/backend/src/metabase_enterprise/entity_retrieval/db.clj
@@ -5,6 +5,7 @@
[metabase.lib.schema.id :as lib.schema.id]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
(mu/defn library-cards-in-collections
@@ -25,7 +26,8 @@
[collection-ids :- [:sequential ::lib.schema.id/collection]
id :- [:maybe ::lib.schema.id/table]]
(t2/select [:model/Table :id :name :display_name :description]
- {:where [:and
+ {:from [(warehouse-schema-overlay/table-query)]
+ :where [:and
[:in :collection_id collection-ids]
[:= :is_published true]
[:= :active true]
diff --git a/enterprise/backend/src/metabase_enterprise/erd/db.clj b/enterprise/backend/src/metabase_enterprise/erd/db.clj
index b3ec7668523..ec3bffc4bf0 100644
--- a/enterprise/backend/src/metabase_enterprise/erd/db.clj
+++ b/enterprise/backend/src/metabase_enterprise/erd/db.clj
@@ -4,6 +4,7 @@
(:require
[metabase.lib.schema.id :as lib.schema.id]
[metabase.util.malli :as mu]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[metabase.warehouse-schema.models.table :as schema.table]
[toucan2.core :as t2]))
@@ -22,7 +23,8 @@
opts :- [:maybe ActiveTablesOpts]]
(let [{:keys [table-ids schema]} opts]
(t2/select :model/Table
- {:select columns
+ {:from [(warehouse-schema-overlay/table-query)]
+ :select columns
:where (cond-> [:and [:= :db_id database-id] [:= :active true]]
(contains? opts :table-ids) (conj [:in :id table-ids])
(contains? opts :schema) (conj (if (= schema "")
@@ -32,7 +34,8 @@
(mu/defn active-fields-for-tables
"The active Fields of the Tables with `table-ids`, in field order."
[table-ids :- [:set ::lib.schema.id/table]]
- (t2/select :model/Field {:where [:and
+ (t2/select :model/Field {:from [(warehouse-schema-overlay/field-query)]
+ :where [:and
[:in :table_id table-ids]
[:= :active true]]
:order-by schema.table/field-order-rule}))
@@ -40,6 +43,7 @@
(mu/defn active-fields
"The active Fields with `field-ids`."
[field-ids :- [:or [:set ::lib.schema.id/field] [:sequential ::lib.schema.id/field]]]
- (t2/select :model/Field {:where [:and
+ (t2/select :model/Field {:from [(warehouse-schema-overlay/field-query)]
+ :where [:and
[:in :id field-ids]
[:= :active true]]}))
diff --git a/enterprise/backend/src/metabase_enterprise/library/db.clj b/enterprise/backend/src/metabase_enterprise/library/db.clj
index 23e013a4a8f..3504f4c6706 100644
--- a/enterprise/backend/src/metabase_enterprise/library/db.clj
+++ b/enterprise/backend/src/metabase_enterprise/library/db.clj
@@ -5,6 +5,7 @@
[metabase.collections.models.collection :as collection]
[metabase.lib.schema.id :as lib.schema.id]
[metabase.util.malli :as mu]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
(mu/defn card-types-in-collections
@@ -15,7 +16,7 @@
(mu/defn published-table-in-collections?
"Whether a published Table exists in the Collections with `collection-ids`."
[collection-ids :- [:sequential ::lib.schema.id/collection]]
- (t2/exists? :model/Table :is_published true :collection_id [:in collection-ids]))
+ (t2/exists? :model/Table :is_published true :collection_id [:in collection-ids] {:from [(warehouse-schema-overlay/table-query)]}))
(mu/defn library-collections
"The readable Library, Library-data, and Library-metrics Collections, ordered by name."
diff --git a/enterprise/backend/src/metabase_enterprise/permission_debug/db.clj b/enterprise/backend/src/metabase_enterprise/permission_debug/db.clj
index 17444ac26a8..dc793fbb982 100644
--- a/enterprise/backend/src/metabase_enterprise/permission_debug/db.clj
+++ b/enterprise/backend/src/metabase_enterprise/permission_debug/db.clj
@@ -5,6 +5,7 @@
[metabase.lib.schema.id :as lib.schema.id]
[metabase.permissions.core :as perms]
[metabase.util.malli :as mu]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
(mu/defn user-superuser?
@@ -20,7 +21,7 @@
(defn- blocked-tables-select
[user-id tables-expr permissions-blocking permissions-granting]
{:select [[:db.name :db_name] :blocked.schema [:blocked.name :table_name] [:pg.name :group_name]]
- :from [[:metabase_table :blocked]]
+ :from [(warehouse-schema-overlay/table-query {:alias :blocked, :user-settings? false})]
:join [[(perms/select-tables-and-groups-granting-perm
{:user-id user-id :is-superuser? false}
permissions-blocking) :perm_grant] [:= :blocked.id :perm_grant.id]
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj
index 625b562ddea..f4ff04727b5 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj
@@ -8,6 +8,7 @@
[metabase.lib.schema.id :as lib.schema.id]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
(def ^:private Conditions
@@ -134,11 +135,15 @@
(t2/select-one model :id id))
(mu/defn instance-with-columns
- "The `columns` of the instance of `model` with `id`, or nil."
+ "The `columns` of the instance of `model` with `id`, or nil; a Table is read through the overlay."
[model :- :keyword
columns :- [:sequential :keyword]
id :- ms/PositiveInt]
- (t2/select-one (into [model] columns) :id id))
+ (t2/select-one (into [model] columns)
+ :id id
+ (if (= model :model/Table)
+ {:from [(warehouse-schema-overlay/table-query)]}
+ {})))
(mu/defn instance-names
"The `:id` and `:name` of the instances of `model` with `ids`."
@@ -178,15 +183,15 @@
:model/Field {:alias "f"
:select [:f.name :f.table_id [:t.collection_id :collection_id] [:t.name :table_name]]
:from [[:metabase_field :f]]
- :join [[:metabase_table :t] [:= :f.table_id :t.id]]}
+ :join [(warehouse-schema-overlay/table-query {:alias :t}) [:= :f.table_id :t.id]]}
:model/Segment {:alias "s"
:select [:s.name :s.table_id [:t.collection_id :collection_id] [:t.name :table_name]]
:from [[:segment :s]]
- :join [[:metabase_table :t] [:= :s.table_id :t.id]]}
+ :join [(warehouse-schema-overlay/table-query {:alias :t}) [:= :s.table_id :t.id]]}
:model/Measure {:alias "s"
:select [:s.name :s.table_id [:t.collection_id :collection_id] [:t.name :table_name]]
:from [[:measure :s]]
- :join [[:metabase_table :t] [:= :s.table_id :t.id]]}))
+ :join [(warehouse-schema-overlay/table-query {:alias :t}) [:= :s.table_id :t.id]]}))
(mu/defn tracking-details-by-id
"The name, table id, collection id, and table name of the `model-key` (Field, Segment, or Measure) instance with
@@ -269,10 +274,15 @@
[card-ids :- [:sequential ::lib.schema.id/card]]
(t2/select [:model/Card :id :type :card_schema] :id [:in card-ids]))
-(mu/defn field-user-settings-exist?
- "Whether the Field with `field-id` has FieldUserSettings."
- [field-id :- ::lib.schema.id/field]
- (t2/exists? :model/FieldUserSettings :field_id field-id))
+(mu/defn user-settings-exist-for-table?
+ "Whether the Table with `table-id`, or any of its Fields, has a user-settings row."
+ [table-id :- ::lib.schema.id/table]
+ (or (t2/exists? :model/TableUserSettings :table_id table-id)
+ (t2/exists? :model/FieldUserSettings
+ {:from [[(t2/table-name :model/FieldUserSettings) :u]]
+ :join [(warehouse-schema-overlay/field-query {:alias :f :user-settings? false})
+ [:= :f.id :u.field_id]]
+ :where [:= :f.table_id table-id]})))
(mu/defn snippets
"The `:id`, `:name`, and `:collection_id` of every NativeQuerySnippet."
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/events.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/events.clj
index 6ceaab9c6bb..88031476314 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/events.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/events.clj
@@ -11,8 +11,8 @@
Tracked model types:
- Card, Dashboard, Document, NativeQuerySnippet, Timeline, Collection
- - Table (when published in a remote-synced collection)
- - Field, Segment (when belonging to a published table in a remote-synced collection)
+ - Table and TableUserSettings, which also carries a Field's own edits (when published in a remote-synced collection)
+ - Segment, Measure (when belonging to a published table in a remote-synced collection)
- Transform, TransformTag, transforms-namespace Collections (when remote-sync-transforms setting is enabled)
- NativeQuerySnippet, snippets-namespace Collections (when Library is remote-synced)"
(:require
@@ -244,7 +244,7 @@
;;; --------------------------------- Spec-based Event Registration (Non-Collection) -----------------------------------
-(doseq [[_model-key model-spec] (dissoc spec/remote-sync-specs :model/Collection :model/Field)]
+(doseq [[_model-key model-spec] (dissoc spec/remote-sync-specs :model/Collection :model/Field :model/Table)]
(register-events-for-spec! model-spec))
;;; ----------------------------------------- Collection Event Handler -------------------------------------------------
@@ -300,27 +300,51 @@
(log/infof "Collection %s no longer needs syncing, marking as removed" (:id object))
(create-or-update-remote-sync-object-entry! "Collection" (:id object) "removed" hydrate-collection-details)))))
-;;; ----------------------------------------- FieldUserSettings Tracking -----------------------------------------------
-;; When a field is updated in a published table, also track any FieldUserSettings row for that field.
-;; FieldUserSettings has no separate event; it piggybacks on :event/field-update.
+;;; ----------------------------------------- TableUserSettings Tracking -----------------------------------------------
+(def ^:private table-spec (get spec/remote-sync-specs :model/Table))
(def ^:private field-spec (get spec/remote-sync-specs :model/Field))
+(defn- hydrate-table-user-settings-details
+ "The RemoteSyncObject details of the Table with `table-id`, as its own `:table_id`/`:table_name`."
+ [table-id]
+ (let [details (spec/hydrate-model-details table-spec table-id)]
+ (assoc details :table_id (:id details) :table_name (:name details))))
+
+(defn- sync-table-user-settings!
+ "Track the Table's TableUserSettings when eligible and any of its user settings exist, else mark it removed."
+ [table-id eligible?]
+ (cond
+ (and eligible? (remote-sync.db/user-settings-exist-for-table? table-id))
+ (create-or-update-remote-sync-object-entry!
+ "TableUserSettings" table-id "update" hydrate-table-user-settings-details)
+
+ (and (not eligible?)
+ (remote-sync.db/rso-exists? "TableUserSettings" table-id))
+ (create-or-update-remote-sync-object-entry!
+ "TableUserSettings" table-id "removed" hydrate-table-user-settings-details)))
+
(events/derive! :event/field-update ::field-update-event)
(events/derive! ::field-update-event :metabase/event)
(methodical/defmethod events/publish-event! ::field-update-event
[_topic {:keys [object]}]
- (let [field-id (:id object)
- eligible? (spec/check-eligibility field-spec object)]
- (cond
- (and eligible? (remote-sync.db/field-user-settings-exist? field-id))
- (create-or-update-remote-sync-object-entry!
- "FieldUserSettings" field-id "update"
- (fn [id] (spec/hydrate-model-details field-spec id)))
-
- (and (not eligible?)
- (remote-sync.db/rso-exists? "FieldUserSettings" field-id))
- (create-or-update-remote-sync-object-entry!
- "FieldUserSettings" field-id "removed"
- (fn [id] (spec/hydrate-model-details field-spec id))))))
+ (when-let [table-id (:table_id object)]
+ (sync-table-user-settings! table-id (spec/check-eligibility field-spec object))))
+
+(defn- handle-table-event!
+ "The generic Table handler plus the Table's TableUserSettings; one handler, since a second primary method on the
+ same events would displace the generic one."
+ [topic {:keys [object] :as event}]
+ (handle-model-event-from-spec table-spec topic event)
+ (sync-table-user-settings! (:id object) (spec/check-eligibility table-spec object)))
+
+(let [event-kws (spec/event-keywords table-spec)
+ parent-kw (:parent event-kws)]
+ (events/derive! parent-kw :metabase/event)
+ (doseq [[_event-type event-kw] (dissoc event-kws :parent)]
+ (events/derive! event-kw parent-kw))
+ (methodical/add-primary-method!
+ #'events/publish-event!
+ parent-kw
+ (fn [topic event] (handle-table-event! topic event))))
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
index 565c75817b8..e381a3269cf 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
@@ -250,7 +250,7 @@
Returns:
- [{:model_type :model_id :path :content_hash}] (entities that fail to serialize are omitted)"
[rows repo-paths]
- (let [storage-opts (serdes/storage-base-context)
+ (let [storage-opts (source/storage-context)
repo-by-eid (into {} (map (fn [{:keys [model_type entity_id path]}] [[model_type entity_id] path])) repo-paths)
serialize (fn [model-type opts id->eid instance]
(try
@@ -271,9 +271,9 @@
(mapcat (fn [[model-type model-rows]]
(let [spec (spec/spec-for-model-type model-type)
model-key (:model-key spec)
- opts {:filter-column :id
- :filter-ids (mapv :model_id model-rows)
- :skip-archived true}
+ opts (merge spec/git-sync-extract-opts
+ {:filter-column :id
+ :filter-ids (mapv :model_id model-rows)})
;; entity-id models: map local id -> entity_id so we can look up the repo path
id->eid (when (and model-key (= :entity-id (:identity spec)))
(remote-sync.db/entity-ids-by-id model-key (mapv :model_id model-rows)))]
@@ -839,11 +839,11 @@
Return:
- [[row entity]] (if no entity, then omit)"
[{:keys [model_type rows]}]
- (let [pk-col (spec/pk-col model_type)
+ (let [pk-col (serdes/primary-key model_type)
id->row (u/index-by :model_id rows)
- opts {:filter-column pk-col
- :filter-ids (mapv :model_id rows)
- :skip-archived true}]
+ opts (merge spec/git-sync-extract-opts
+ {:filter-column pk-col
+ :filter-ids (mapv :model_id rows)})]
;; extract-one must run inside the extract-query reduction, while its ResultSet is open
(into [] (keep (fn [instance]
(when-let [row (id->row (get instance pk-col))]
@@ -989,7 +989,7 @@
- {:writes [{:id :model_type :model_id :file_path}] :delete-paths [path] :removed-ids [id]}, or
- :remote-sync/incremental-not-possible when any row can't go incrementally"
[snapshot rows]
- (let [opts (serdes/storage-base-context)
+ (let [opts (source/storage-context)
;; create/update rows on entity-id models need an entity (extracted per chunk); everything else
;; (removed/delete, non-entity-id, bad status) is decided with no entity
{cu-rows true other-rows false} (group-by #(boolean (and (#{"create" "update"} (:status %))
@@ -1104,7 +1104,7 @@
total (count export-rows)
span (- export-progress-serialize export-progress-plan-done)]
(report export-progress-plan-done {:force? true})
- (let [opts (serdes/storage-base-context)
+ (let [opts (source/storage-context)
[synced version] (commit-staged! snapshot message
(fn [commit]
(source.p/replace-all! commit) ; replace the managed dirs wholesale
@@ -1136,7 +1136,7 @@
total (max 1 (count writes))
span (- export-progress-serialize export-progress-plan-done)]
(report export-progress-plan-done {:force? true})
- (let [opts (serdes/storage-base-context)
+ (let [opts (source/storage-context)
[synced version] (commit-staged! snapshot message
(fn [commit]
(let [synced (stage-writes commit opts writes
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj
index 4b6ecc55806..5df7f6a69ad 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj
@@ -68,6 +68,11 @@
(atom nil) (atom []))
(seq root-dependencies) (ingestable/wrap-root-dep-ingestable root-dependencies)))
+(defn storage-context
+ "The serdes storage context for git sync: a Table's user settings file stands in for the Table's own."
+ []
+ (assoc (serdes/storage-base-context) :inline-user-settings true))
+
(defn entity->path
"The repo-relative path an extracted `entity` serializes to, using storage context `opts`."
[opts entity]
@@ -84,7 +89,7 @@
(defn entity->file-spec
"Serializes a single extracted entity into a `{:path :content}` file spec, using storage context
- `opts` (from `serdes/storage-base-context`)."
+ `opts` (from [[storage-context]])."
[opts entity]
{:path (entity->path opts entity)
:content (entity->content entity)})
@@ -99,7 +104,7 @@
can't be extracted. Hashes the live DB serialization (never on-disk bytes), so it's stable across sync points."
[row]
(when-let [entity (first (spec/extract-entities-for-rows [row]))]
- (content-hash (:content (entity->file-spec (serdes/storage-base-context) entity)))))
+ (content-hash (:content (entity->file-spec (storage-context) entity)))))
(defn serialize-specs
"Serializes a stream of entities into an eager vector of `{:path :content}` file specs. Reports progress
@@ -108,7 +113,7 @@
Throws Exception if any entity in the stream is an Exception instance."
[stream task-id]
- (let [opts (serdes/storage-base-context)
+ (let [opts (storage-context)
stream-count (bounded-count 10000 stream)]
(into []
(map-indexed (fn [idx entity]
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
index ecd6b1d9f3e..7cd9839ab5d 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
@@ -819,15 +819,14 @@
;;; ---------------------------------------------------- Hydration -----------------------------------------------------
(defn hydrate-model-details
- "Hydrates model details for RemoteSyncObject based on spec.
- Returns a map with the fields needed to populate the sync object."
+ "Returns a map with the fields (always including :id) needed to populate a RemoteSyncObject for `spec`."
[{:keys [model-key tracking]} model-id]
(if (:hydrate-query? tracking)
;; Use the tracking-join query for models needing a Table join (Field, Segment, Measure)
(remote-sync.db/tracking-details-by-id model-key model-id)
;; Simple select using select-fields
(when-let [fields (:select-fields tracking)]
- (remote-sync.db/instance-with-columns model-key fields model-id))))
+ (remote-sync.db/instance-with-columns model-key (into [:id] fields) model-id))))
(defn build-sync-object-fields
"Builds the fields map for RemoteSyncObject from hydrated model details.
@@ -1136,14 +1135,17 @@
(defmethod query-export-roots :default [_] nil)
-(def ^:private git-sync-extract-opts
- "Serdes extraction opts for git sync. :user-edits-only restricts field-level export to
- FieldUserSettings (user-curated metadata only), omitting the full Field YAML for every column."
+(def git-sync-extract-opts
+ "Serdes extraction opts for git sync."
{:include-field-values false
:include-database-secrets false
:continue-on-error false
:skip-archived true
- :user-edits-only true})
+ :inline-user-settings true})
+
+(def ^:private models-traversed-but-not-stored
+ "Models git sync walks through but never writes."
+ #{"Table" "Field"})
(defn exportable-entities
"What a full export would serialize: a map of {model-name [id ...]} — the export roots plus their transitive
@@ -1155,12 +1157,7 @@
targets (-> #{}
(into (keys (u/traverse root-targets #(serdes/descendants (first %) (second %) git-sync-extract-opts))))
(into (keys (u/traverse root-targets #(serdes/required (first %) (second %))))))]
- (u/group-by first second targets)))
-
-(defn pk-col
- "Returns the PK column keyword for `model`. FieldUserSettings uses :field_id; all others use :id."
- [model]
- (if (= model "FieldUserSettings") :field_id :id))
+ (apply dissoc (u/group-by first second targets) models-traversed-but-not-stored)))
(defn extract-entities-for-export
"Extracts all entities for remote-sync export based on enabled specs.
@@ -1173,9 +1170,9 @@
3. Are in one of the provided collections (or descendants)"
[]
(eduction (map (fn [[model ids]]
- (serdes/extract-all model {:filter-column (pk-col model)
- :filter-ids (vec ids)
- :skip-archived true})))
+ (serdes/extract-all model (merge git-sync-extract-opts
+ {:filter-column (serdes/primary-key model)
+ :filter-ids (vec ids)}))))
cat
(exportable-entities)))
@@ -1186,8 +1183,8 @@
[rows]
(let [by-model (u/group-by :model_type :model_id conj #{} rows)]
(eduction (map (fn [[model ids]]
- (serdes/extract-all model {:filter-column (pk-col model)
- :filter-ids (vec ids)
- :skip-archived true})))
+ (serdes/extract-all model (merge git-sync-extract-opts
+ {:filter-column (serdes/primary-key model)
+ :filter-ids (vec ids)}))))
cat
by-model)))
diff --git a/enterprise/backend/src/metabase_enterprise/replacement/db.clj b/enterprise/backend/src/metabase_enterprise/replacement/db.clj
index c97dc8ee7ab..2c8154595b8 100644
--- a/enterprise/backend/src/metabase_enterprise/replacement/db.clj
+++ b/enterprise/backend/src/metabase_enterprise/replacement/db.clj
@@ -13,7 +13,7 @@
[metabase.util.honey-sql-2 :as h2x]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
- [metabase.warehouse-schema.schema :as warehouse-schema.schema]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
(mu/defn run
@@ -75,7 +75,7 @@
(mu/defn tables-with-ids
"The Tables with `ids`."
[ids :- [:sequential ms/PositiveInt]]
- (t2/select :model/Table :id [:in ids]))
+ (t2/select :model/Table :id [:in ids] {:from [(warehouse-schema-overlay/table-query)]}))
(mu/defn dashboards-with-ids
"The Dashboards with `ids`."
@@ -182,28 +182,22 @@
(mu/defn table-database-id
"The Database ID of the Table with `table-id`."
[table-id :- ::lib.schema.id/table]
- (t2/select-one-fn :db_id :model/Table :id table-id))
+ (t2/select-one-fn :db_id :model/Table :id table-id {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))
(mu/defn active-fields-of-table
"The active Fields of the Table with `table-id`."
[table-id :- ::lib.schema.id/table]
- (t2/select :model/Field :table_id table-id :active true))
+ (t2/select :model/Field :table_id table-id :active true {:from [(warehouse-schema-overlay/field-query)]}))
(mu/defn active-field-ids-of-table
"The IDs of the active Fields of the Table with `table-id`."
[table-id :- ::lib.schema.id/table]
- (t2/select-pks-set :model/Field :table_id table-id :active true))
+ (t2/select-pks-set :model/Field :table_id table-id :active true {:from [(warehouse-schema-overlay/field-query {:user-settings? false})]}))
(mu/defn active-fk-to-fields-exists?
"Whether an active Field points at one of the Fields with `field-ids`."
[field-ids :- [:set ::lib.schema.id/field]]
- (t2/exists? :model/Field :fk_target_field_id [:in field-ids] :active true))
-
-(mu/defn update-field!
- "Apply `changes` to the Field with `field-id`, returning the number updated."
- [field-id :- ::lib.schema.id/field
- changes :- ::warehouse-schema.schema/field.update]
- (t2/update! :model/Field field-id changes))
+ (t2/exists? :model/Field :fk_target_field_id [:in field-ids] :active true {:from [(warehouse-schema-overlay/field-query)]}))
(mu/defn sandbox-exists-for-table?
"Whether a Sandbox is defined on the Table with `table-id`."
diff --git a/enterprise/backend/src/metabase_enterprise/replacement/runner.clj b/enterprise/backend/src/metabase_enterprise/replacement/runner.clj
index 005b0cbb5b5..6fae87f12bf 100644
--- a/enterprise/backend/src/metabase_enterprise/replacement/runner.clj
+++ b/enterprise/backend/src/metabase_enterprise/replacement/runner.clj
@@ -169,8 +169,8 @@
[:description :display_name :semantic_type :fk_target_field_id :settings :visibility_type])
(defn- copy-model-metadata-overrides!
- "Copy user-edited metadata from a model's result_metadata onto the Fields of the
- output table. Writes to both Field and FieldUserSettings so overrides survive sync."
+ "Copy user-edited metadata from a model's result_metadata onto the FieldUserSettings of the
+ output table's Fields, so the overrides keep showing on the new Fields after the swap."
[card-id table-id]
(let [card (replacement.db/card card-id)
result-metadata (:result_metadata card)
@@ -180,7 +180,6 @@
:let [field (field-by-name (source-swap.util/column-match-key col-meta))
overrides (u/select-keys-when col-meta :non-nil metadata-override-keys)]
:when (and field (seq overrides))]
- (replacement.db/update-field! (:id field) overrides)
(field-user-settings/upsert-user-settings field overrides))))
(defn run-swap-model-with-transform!
diff --git a/enterprise/backend/src/metabase_enterprise/sandbox/db.clj b/enterprise/backend/src/metabase_enterprise/sandbox/db.clj
index 8906fb05e7c..7528107c562 100644
--- a/enterprise/backend/src/metabase_enterprise/sandbox/db.clj
+++ b/enterprise/backend/src/metabase_enterprise/sandbox/db.clj
@@ -10,6 +10,7 @@
[metabase.users.schema :as users.schema]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
(mu/defn sandbox
@@ -60,7 +61,8 @@
(t2/select :model/Sandbox
{:select [:s.group_id :s.table_id :t.db_id :t.schema]
:from [[:sandboxes :s]]
- :join [[:metabase_table :t] [:= :s.table_id :t.id]]
+ :join [(warehouse-schema-overlay/table-query {:alias :t, :user-settings? false})
+ [:= :s.table_id :t.id]]
:where [:and
(when group-id [:= :s.group_id group-id])
(when group-ids [:in :s.group_id group-ids])
@@ -79,7 +81,7 @@
[:table.db_id :db_id]
[:table.schema :schema]]
:from [[:sandboxes]]
- :left-join [[:metabase_table :table]
+ :left-join [(warehouse-schema-overlay/table-query {:alias :table, :user-settings? false})
[:= :sandboxes.table_id :table.id]]
:where [:and
[:in :sandboxes.group_id group-ids]
@@ -149,7 +151,7 @@
(mu/defn table
"The Table with `table-id`, or nil."
[table-id :- ::lib.schema.id/table]
- (t2/select-one :model/Table :id table-id))
+ (t2/select-one :model/Table :id table-id {:from [(warehouse-schema-overlay/table-query)]}))
(mu/defn tables-of-database
"The `:id`, `:db_id`, and `:schema` of the Tables of the Database with `db-id`, restricted to `schema` when
@@ -158,7 +160,8 @@
schema-only? :- :boolean
schema :- [:maybe :string]]
(t2/select [:model/Table :id :db_id :schema]
- {:where [:and
+ {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]
+ :where [:and
[:= :db_id db-id]
(when schema-only?
[:= :schema schema])]}))
@@ -168,14 +171,14 @@
[table-id :- ::lib.schema.id/table]
(t2/select-one :model/Database
:id ^:allow-subquery {:select [:t.db_id]
- :from [[(t2/table-name :model/Table) :t]]
+ :from [(warehouse-schema-overlay/table-query {:alias :t, :user-settings? false})]
:where [:= :t.id table-id]}))
(mu/defn fields-of-table-named
"The `:id` and `:name` of the Fields of the Table with `table-id` named one of `field-names`."
[table-id :- ::lib.schema.id/table
field-names :- [:set :string]]
- (t2/select [:model/Field :id :name] :table_id table-id :name [:in field-names]))
+ (t2/select [:model/Field :id :name] :table_id table-id :name [:in field-names] {:from [(warehouse-schema-overlay/field-query {:user-settings? false})]}))
(mu/defn cards-by-id
"A map of Card ID to the query, result metadata, and schema of the Cards with `card-ids`."
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/db.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/db.clj
index 536e30986c6..0cc746fa705 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/db.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/db.clj
@@ -10,6 +10,7 @@
[metabase.search.scoring :as search.scoring]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
(mu/defn library-root-collections
@@ -27,7 +28,8 @@
authoritative."
[]
(t2/reducible-select [:model/Table :id :is_published :data_layer :data_authority]
- {:where [:and
+ {:from [(warehouse-schema-overlay/table-query)]
+ :where [:and
[:= :active true]
[:or [:= :is_published true]
[:= :data_authority ^:allow-raw-sql [:inline "authoritative"]]]]}))
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj
index b05bf83dcf9..e5ee7ba20ff 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj
@@ -166,11 +166,9 @@
;; so cards that reference them can still be exported and imported correctly
(and analytics-cards (contains? by-model "Card"))
(update "Card" (fn [ids] (vec (remove analytics-cards ids)))))
- ;; FieldUserSettings has a non-standard PK (field_id, not id) — use the right column.
- pk-col (fn [model] (if (= model "FieldUserSettings") :field_id :id))
extract-by-ids (fn [[model ids]]
(serdes/extract-all model (merge opts {:collection-set coll-set
- :filter-column (pk-col model)
+ :filter-column (serdes/primary-key model)
:filter-ids (vec ids)})))
extract-all (fn [model]
(serdes/extract-all model (assoc opts :collection-set coll-set)))]
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
index 278b3e82db7..f7a80dd154d 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
@@ -5,19 +5,18 @@
"Schema model types"
["Database"
"Field"
- "FieldUserSettings"
"Measure"
"Segment"
"Table"
+ "TableUserSettings"
"Channel"])
(def data-model-in-collection
- "Data model types that can be found in collections (via published tables).
- These are extracted by ID when discovered via descendants, even if no-data-model is set.
- Includes both Field (full serdes) and FieldUserSettings (user-edits-only / git sync)."
+ "Data model types that can be found in collections (via published tables), extracted by ID when discovered via
+ descendants even if no-data-model is set."
["Table"
+ "TableUserSettings"
"Field"
- "FieldUserSettings"
"Segment"])
(def content
@@ -53,6 +52,7 @@
["DashboardCard"
"DashboardTab"
"Dimension"
+ "FieldUserSettings"
"ParameterCard"
"DashboardCardSeries"
"MetabotPrompt"
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_inspector/db.clj b/enterprise/backend/src/metabase_enterprise/transforms_inspector/db.clj
index 53a61b69740..1c0c94f98c5 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_inspector/db.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_inspector/db.clj
@@ -4,12 +4,13 @@
(:require
[metabase.lib.schema.id :as lib.schema.id]
[metabase.util.malli :as mu]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
(mu/defn table-source-rows
"The ID, name, schema, and Database ID of the Tables with `table-ids`, as source info."
[table-ids :- [:or [:set ::lib.schema.id/table] [:sequential ::lib.schema.id/table]]]
- (t2/select [:model/Table [:id :table-id] [:name :table-name] :schema [:db_id :db-id]] :id [:in table-ids]))
+ (t2/select [:model/Table [:id :table-id] [:name :table-name] :schema [:db_id :db-id]] :id [:in table-ids] {:from [(warehouse-schema-overlay/table-query)]}))
(mu/defn database-engine
"The engine of the Database with `database-id`."
@@ -19,4 +20,4 @@
(mu/defn active-fields-for-table
"The active Fields of the Table with `table-id`."
[table-id :- ::lib.schema.id/table]
- (t2/select :model/Field :table_id table-id :active true))
+ (t2/select :model/Field :table_id table-id :active true {:from [(warehouse-schema-overlay/field-query)]}))
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/db.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/db.clj
index f65ceabc769..0327d051352 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/db.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/db.clj
@@ -6,17 +6,18 @@
[metabase.lib.schema.id :as lib.schema.id]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
(mu/defn table-database-ids
"The set of Database IDs of the Tables with `table-ids`."
[table-ids :- [:sequential ::lib.schema.id/table]]
- (t2/select-fn-set :db_id [:model/Table :db_id] :id [:in table-ids]))
+ (t2/select-fn-set :db_id [:model/Table :db_id] :id [:in table-ids] {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))
(mu/defn table-database-id
"The Database ID of the raw table row with `table-id`."
[table-id :- ::lib.schema.id/table]
- (t2/select-one-fn :db_id (t2/table-name :model/Table) :id table-id))
+ (t2/select-one-fn :db_id (t2/table-name :model/Table) :id table-id {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))
(mu/defn database
"The Database with `database-id`, or nil."
@@ -66,4 +67,5 @@
;; we are only interested in top-level objects, so filter out nested fields (parent or path)
:parent_id nil
:nfc_path nil
- {:order-by [[:database_position :asc]]}))
+ {:from [(warehouse-schema-overlay/field-query)]
+ :order-by [[:database_position :asc]]}))
diff --git a/enterprise/backend/src/metabase_enterprise/upload_management/db.clj b/enterprise/backend/src/metabase_enterprise/upload_management/db.clj
index 7f4a8092795..e4f123e5a1c 100644
--- a/enterprise/backend/src/metabase_enterprise/upload_management/db.clj
+++ b/enterprise/backend/src/metabase_enterprise/upload_management/db.clj
@@ -4,6 +4,7 @@
(:require
[metabase.lib.schema.id :as lib.schema.id]
[metabase.util.malli :as mu]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
(mu/defn attached-dwh-database-id
@@ -14,14 +15,15 @@
(mu/defn non-upload-tables-for-database
"The active Tables of the Database with `database-id` that were not uploaded."
[database-id :- ::lib.schema.id/database]
- (t2/select :model/Table :db_id database-id :active true :is_upload false))
+ (t2/select :model/Table :db_id database-id :active true :is_upload false {:from [(warehouse-schema-overlay/table-query)]}))
(mu/defn upload-tables
"The active uploaded Tables, ordered by name."
[]
- (t2/select :model/Table :active true :is_upload true {:order-by [[:name :asc]]}))
+ (t2/select :model/Table :active true :is_upload true {:from [(warehouse-schema-overlay/table-query)]
+ :order-by [[:name :asc]]}))
(mu/defn table
"The Table with `table-id`, or nil."
[table-id :- ::lib.schema.id/table]
- (t2/select-one :model/Table table-id))
+ (t2/select-one :model/Table :id table-id {:from [(warehouse-schema-overlay/table-query)]}))
diff --git a/enterprise/backend/test/metabase_enterprise/action_v2/db_test.clj b/enterprise/backend/test/metabase_enterprise/action_v2/db_test.clj
new file mode 100644
index 00000000000..cdd1c6b1efb
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/action_v2/db_test.clj
@@ -0,0 +1,30 @@
+(ns metabase-enterprise.action-v2.db-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase-enterprise.action-v2.db :as action-v2.db]
+ [metabase.test :as mt]))
+
+(deftest pk-fields-for-table-honors-user-set-semantic-type-test
+ (testing "pk-fields-for-table honors a user-set semantic_type that differs from the sync value"
+ (mt/with-temp [:model/Table {table-id :id} {}
+ :model/Field {field-id :id} {:table_id table-id, :semantic_type nil}]
+ (testing "sanity check: not a PK by sync"
+ (is (= [] (map :id (action-v2.db/pk-fields-for-table table-id)))))
+ (mt/with-temp [:model/FieldUserSettings _ {:field_id field-id
+ :semantic_type :type/PK
+ :semantic_type_set true}]
+ (is (= [field-id] (map :id (action-v2.db/pk-fields-for-table table-id))))))))
+
+(deftest category-list-field-ids-by-name-honors-user-settings-test
+ (testing "category-list-field-ids-by-name honors user-set has_field_values/semantic_type"
+ (mt/with-temp [:model/Table {table-id :id} {}
+ :model/Field {field-id :id} {:table_id table-id, :name "STATUS"
+ :has_field_values nil, :semantic_type nil}]
+ (testing "sanity check: not a category-list field by sync"
+ (is (= [] (action-v2.db/category-list-field-ids-by-name table-id ["status"]))))
+ (mt/with-temp [:model/FieldUserSettings _ {:field_id field-id
+ :has_field_values :list
+ :semantic_type :type/Category
+ :semantic_type_set true}]
+ (is (= [{:id field-id, :lower_name "status"}]
+ (action-v2.db/category-list-field-ids-by-name table-id ["status"])))))))
diff --git a/enterprise/backend/test/metabase_enterprise/data_studio/api/table_test.clj b/enterprise/backend/test/metabase_enterprise/data_studio/api/table_test.clj
index 9a012c96ead..61a467a96b1 100644
--- a/enterprise/backend/test/metabase_enterprise/data_studio/api/table_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/data_studio/api/table_test.clj
@@ -11,12 +11,29 @@
[metabase.permissions.models.permissions-group :as perms-group]
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
(use-fixtures :once (fixtures/initialize :db))
+(defn- user-table
+ "The Table with `table-id` as users see it. A bare `t2/select :model/Table` shows what sync wrote."
+ [table-id]
+ (t2/select-one :model/Table :id table-id {:from [(warehouse-schema-overlay/table-query)]}))
+
+(defn- user-tables
+ "The Tables with `table-ids` as users see it, ordered by `order-by`."
+ [table-ids order-by]
+ (t2/select :model/Table :id [:in table-ids] {:from [(warehouse-schema-overlay/table-query)]
+ :order-by [order-by]}))
+
+(defn- user-table-fn
+ "The value of `column` on the Table with `table-id` as users see it."
+ [column table-id]
+ (get (user-table table-id) column))
+
(deftest table-metadata-survives-deleted-transform-test
(testing "a table whose creating transform has since been deleted (metabase#69904)"
(mt/with-premium-features #{:transforms-basic :hosting}
@@ -24,7 +41,7 @@
:model/Table {table-id :id} {:transform_id (:id transform)}]
(t2/delete! :model/Transform (:id transform))
(testing "metabase_table.transform_id -> transform.id is ON DELETE SET NULL, not left dangling"
- (is (nil? (t2/select-one-fn :transform_id :model/Table :id table-id))))
+ (is (nil? (user-table-fn :transform_id table-id))))
(testing "GET /api/table/:id does not crash"
(is (=? {:id table-id} (mt/user-http-request :crowberto :get 200 (str "table/" table-id)))))
(testing "GET /api/table (list, which hydrates :transform when transforms are enabled) does not crash"
@@ -52,7 +69,7 @@
{:display_name "Venues"
:collection_id collection-id
:is_published true}]
- (t2/select :model/Table :id [:in [(mt/id :users) (mt/id :venues)]] {:order-by [:display_name]}))))
+ (user-tables [(mt/id :users) (mt/id :venues)] :display_name))))
(testing "audit log entries are created for publish"
(is (=? {:topic :table-publish, :model "Table", :model_id (mt/id :users)}
(mt/latest-audit-log-entry "table-publish" (mt/id :users))))
@@ -67,7 +84,7 @@
(is (=? {:display_name "Venues"
:collection_id nil
:is_published false}
- (t2/select-one :model/Table (mt/id :venues))))
+ (user-table (mt/id :venues))))
(testing "audit log entry is created for unpublish"
(is (=? {:topic :table-unpublish, :model "Table", :model_id (mt/id :venues)}
(mt/latest-audit-log-entry "table-unpublish" (mt/id :venues)))))))))
@@ -75,7 +92,7 @@
(is (=? {:display_name "Users"
:collection_id nil
:is_published false}
- (t2/select-one :model/Table (mt/id :users))))))
+ (user-table (mt/id :users))))))
(testing "returns 404 when no library-data collection exists"
(is (= "Not found."
(mt/user-http-request :crowberto :post 404 "ee/data-studio/table/publish-tables"
@@ -99,7 +116,7 @@
:collection_id subcollection-id})
(is (=? {:collection_id subcollection-id
:is_published true}
- (t2/select-one :model/Table (mt/id :users)))))))))
+ (user-table (mt/id :users)))))))))
(deftest publishing-info-test
(mt/with-premium-features #{:library :audit-app}
@@ -151,13 +168,17 @@
(without-library
(mt/with-temp [:model/Collection {collection-id :id} {:type collection/library-data-collection-type}]
;; a bumped updated_at proves the write went through the Toucan pipeline (:hook/timestamped?),
- ;; the same pipeline that fires :hook/search-index; a raw UPDATE would leave it unchanged
(let [baseline (t/offset-date-time 2020)
;; raw update, precisely to keep the timestamped hook from overwriting the backdate
- backdate! #(t2/query {:update (t2/table-name :model/Table)
- :set {:updated_at baseline}
- :where [:= :id (mt/id :venues)]})
- updated-at #(t2/select-one-fn :updated_at :model/Table (mt/id :venues))]
+ backdate! #(do (t2/query {:update (t2/table-name :model/TableUserSettings)
+ :set {:updated_at baseline}
+ :where [:= :table_id (mt/id :venues)]})
+ (t2/query {:update (t2/table-name :model/Table)
+ :set {:updated_at baseline}
+ :where [:= :id (mt/id :venues)]}))
+ updated-at #(or (t2/select-one-fn :updated_at :model/TableUserSettings
+ :table_id (mt/id :venues))
+ (user-table-fn :updated_at (mt/id :venues)))]
(backdate!)
(mt/user-http-request :crowberto :post 200 "ee/data-studio/table/publish-tables"
{:table_ids [(mt/id :venues)]
@@ -245,7 +266,7 @@
(mt/user-http-request :crowberto :post 200 "ee/data-studio/table/publish-tables"
{:table_ids [orders-id]
:collection_id collection-id})
- (are [table-id] (true? (t2/select-one-fn :is_published :model/Table table-id))
+ (are [table-id] (true? (user-table-fn :is_published table-id))
orders-id products-id))))))
(deftest publish-tables-does-not-move-already-published-upstream-test
@@ -275,10 +296,10 @@
:collection_id coll-y})
(testing "the selected table is published into the target collection"
(is (=? {:is_published true :collection_id coll-y}
- (t2/select-one [:model/Table :is_published :collection_id] orders-id))))
+ (user-table orders-id))))
(testing "the already-published upstream table stays in its original collection"
(is (=? {:is_published true :collection_id coll-x}
- (t2/select-one [:model/Table :is_published :collection_id] products-id))))))))
+ (user-table products-id))))))))
(deftest publish-tables-recursive-upstream-test
(mt/with-premium-features #{:library}
@@ -316,7 +337,7 @@
(mt/user-http-request :crowberto :post 200 "ee/data-studio/table/publish-tables"
{:table_ids [items-id]
:collection_id collection-id})
- (are [table-id] (true? (t2/select-one-fn :is_published :model/Table table-id))
+ (are [table-id] (true? (user-table-fn :is_published table-id))
items-id orders-id purchasers-id))))))
(deftest unpublish-tables-with-downstream-dependents-test
@@ -345,7 +366,7 @@
(testing "unpublishing products also unpublishes orders (downstream dependent)"
(mt/user-http-request :crowberto :post 204 "ee/data-studio/table/unpublish-tables"
{:table_ids [products-id]})
- (are [table-id] (false? (t2/select-one-fn :is_published :model/Table table-id))
+ (are [table-id] (false? (user-table-fn :is_published table-id))
products-id orders-id))))))
(deftest unpublish-tables-recursive-downstream-test
@@ -386,7 +407,7 @@
(testing "unpublishing customers also unpublishes orders and order_items (recursive downstream)"
(mt/user-http-request :crowberto :post 204 "ee/data-studio/table/unpublish-tables"
{:table_ids [purchasers-id]})
- (are [table-id] (false? (t2/select-one-fn :is_published :model/Table table-id))
+ (are [table-id] (false? (user-table-fn :is_published table-id))
purchasers-id orders-id items-id))))))
(defn- with-fk-linked-published-tables
@@ -418,9 +439,9 @@
(fn [{:keys [coll-a x-id y-id]}]
(t2/delete! :model/Collection :id coll-a)
(testing "Table X (in the deleted collection) is unpublished"
- (is (=? {:is_published false :collection_id nil} (t2/select-one :model/Table x-id))))
+ (is (=? {:is_published false :collection_id nil} (user-table x-id))))
(testing "Table Y (FK-linked, in another collection) is also unpublished"
- (is (=? {:is_published false :collection_id nil} (t2/select-one :model/Table y-id)))))))))
+ (is (=? {:is_published false :collection_id nil} (user-table y-id)))))))))
(deftest unpublish-fk-linked-tables-on-collection-archive-test
(mt/with-premium-features #{:library}
@@ -431,9 +452,9 @@
(collection/archive-or-unarchive-collection!
(t2/select-one :model/Collection :id coll-a) {:archived true}))
(testing "Table X (in the archived collection) is unpublished"
- (is (=? {:is_published false :collection_id nil} (t2/select-one :model/Table x-id))))
+ (is (=? {:is_published false :collection_id nil} (user-table x-id))))
(testing "Table Y (FK-linked, in another collection) is also unpublished"
- (is (=? {:is_published false :collection_id nil} (t2/select-one :model/Table y-id)))))))))
+ (is (=? {:is_published false :collection_id nil} (user-table y-id)))))))))
;;; ------------------------------------------ Publish/Unpublish requires write and query perms ------------------------------------------
diff --git a/enterprise/backend/test/metabase_enterprise/data_studio/permissions/query_test.clj b/enterprise/backend/test/metabase_enterprise/data_studio/permissions/query_test.clj
index 549df73f932..8632090ba5a 100644
--- a/enterprise/backend/test/metabase_enterprise/data_studio/permissions/query_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/data_studio/permissions/query_test.clj
@@ -11,6 +11,7 @@
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
[metabase.util :as u]
+ [metabase.warehouse-schema.models.table-user-settings :as schema.table-user-settings]
[toucan2.core :as t2]))
(use-fixtures :once (fixtures/initialize :db))
@@ -39,7 +40,9 @@
(mt/with-temp [:model/User {user-id :id} {}
:model/Collection {collection-id :id} {:type "library-data"}]
(try
- (t2/update! :model/Table (mt/id :venues) {:is_published true :collection_id collection-id})
+ (schema.table-user-settings/upsert-user-settings (t2/select-one :model/Table (mt/id :venues))
+ {:is_published true
+ :collection_id collection-id})
(let [all-users (perms/all-users-group)]
(perms/set-database-permission! all-users (mt/id) :perms/view-data db-view-data)
(perms/set-database-permission! all-users (mt/id) :perms/create-queries :no)
@@ -50,7 +53,8 @@
(perms/set-table-permission! all-users (mt/id :venues) :perms/create-queries :no))
(f user-id collection-id)
(finally
- (t2/update! :model/Table (mt/id :venues) {:is_published false :collection_id nil})))))))
+ (schema.table-user-settings/unset-user-settings! (t2/select-one :model/Table (mt/id :venues))
+ [:is_published :collection_id])))))))
(deftest published-table-test
(testing "Published tables grant query access via collection permissions only with library enabled\n"
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/content_hash_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/content_hash_test.clj
index 74cc21f5d77..c59b149cf51 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/content_hash_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/content_hash_test.clj
@@ -199,7 +199,7 @@
(deftest content-metadata-matches-row-hash-test
(testing "a full export writes a content_hash matching per-row row->content-hash for every identity flavor —
including the extract-query overrides (Collection, NativeQuerySnippet), hybrids (Measure), and path
- models (Field) that have no entity_id (GHY-3933)"
+ models (TableUserSettings, keyed by table_id) that have no entity_id (GHY-3933)"
(with-library-synced
(mt/with-temporary-setting-values [remote-sync-type :read-write]
(mt/with-temp [:model/Database db {:name "DB"}
@@ -209,14 +209,14 @@
:model/Card card {:name "Card" :dataset_query (mt/mbql-query venues) :collection_id (:id rs)}
:model/NativeQuerySnippet snip {:name "Snip" :content "SELECT 1" :collection_id (:id snips)}
:model/Table table {:name "T" :schema "PUBLIC" :db_id (:id db) :is_published true :collection_id (:id data)}
- :model/Field field {:name "F" :table_id (:id table) :base_type :type/Integer}
- :model/FieldUserSettings _ {:field_id (:id field) :description "curated"}
+ :model/Field _field {:name "F" :table_id (:id table) :base_type :type/Integer}
+ :model/TableUserSettings _ {:table_id (:id table) :display_name "curated"}
:model/Measure measure {:name "M" :table_id (:id table)}]
(mt/with-model-cleanup [:model/RemoteSyncTask]
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})
rows (mapv (fn [[mt id]] {:model_type mt :model_id id})
[["Card" (:id card)] ["Collection" (:id rs)] ["NativeQuerySnippet" (:id snip)]
- ["FieldUserSettings" (:id field)] ["Measure" (:id measure)]])]
+ ["TableUserSettings" (:id table)] ["Measure" (:id measure)]])]
(t2/delete! :model/RemoteSyncObject)
(doseq [row rows]
(t2/insert! :model/RemoteSyncObject (merge row {:model_name "x" :status "synced"
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.clj
index 06f3221a0c4..9359f7094c5 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.clj
@@ -1117,10 +1117,6 @@
;;; Field Event Tests
-;; Field RSO tracking has been replaced by FieldUserSettings RSO tracking.
-;; field-update events now track FieldUserSettings (user-curated metadata) rather than raw Field rows.
-;; See the FieldUserSettings RSO Tracking section below for the current behavior tests.
-
(deftest field-update-event-no-entry-for-unpublished-table-test
(testing "field-update event creates no RSOs for an unpublished table"
(mt/with-temp [:model/Collection remote-sync-collection {:is_remote_synced true :name "Remote-Sync" :type "library-data"}
@@ -1149,12 +1145,11 @@
{:object field :user-id (mt/user->id :rasta)})
(is (empty? (t2/select :model/RemoteSyncObject))))))
-;;; ----------------------------------------- FieldUserSettings RSO Tracking ------------------------------------------
-;; In git-sync mode, field updates should track FieldUserSettings RSOs only — not Field RSOs.
-;; Field YAMLs are replaced by FieldUserSettings YAMLs (user-curated metadata only).
+;;; ----------------------------------------- TableUserSettings RSO Tracking from Field Events --------------------------
-(deftest field-update-creates-only-fus-rso-when-fus-exists-test
- (testing "field-update on an eligible field with a FUS row creates ONLY a FieldUserSettings RSO — no Field RSO"
+(deftest field-update-creates-table-user-settings-rso-when-fus-exists-test
+ (testing "field-update on an eligible field with a FUS row creates a TableUserSettings RSO for its Table — no
+ Field or FieldUserSettings RSO"
(mt/with-temp [:model/Collection coll {:is_remote_synced true :name "Remote-Sync" :type "library-data"}
:model/Table table {:name "T" :is_published true :collection_id (:id coll)}
:model/Field field {:name "f" :table_id (:id table) :base_type :type/Text}]
@@ -1162,9 +1157,9 @@
(t2/delete! :model/RemoteSyncObject)
(events/publish-event! :event/field-update {:object field :user-id (mt/user->id :rasta)})
(let [entries (t2/select :model/RemoteSyncObject)]
- (is (= 1 (count entries)) "exactly one RSO — no Field RSO alongside the FUS RSO")
- (is (=? {:model_type "FieldUserSettings"
- :model_id (:id field)
+ (is (= 1 (count entries)) "exactly one RSO — the Table's TableUserSettings, no Field/FieldUserSettings RSO")
+ (is (=? {:model_type "TableUserSettings"
+ :model_id (:id table)
:status "update"
:model_table_id (:id table)}
(first entries)))))))
@@ -1272,14 +1267,15 @@
(assert-removal-survived coll-id card)))))
(deftest field-update-no-rso-when-no-fus-row-test
- (testing "field-update on an eligible field with NO FUS row creates no RSOs at all"
+ (testing "field-update on an eligible field with NO FUS row, and no TableUserSettings row either, creates no
+ RSOs at all"
(mt/with-temp [:model/Collection coll {:is_remote_synced true :name "Remote-Sync" :type "library-data"}
:model/Table table {:name "T" :is_published true :collection_id (:id coll)}
:model/Field field {:name "f" :table_id (:id table) :base_type :type/Text}]
(t2/delete! :model/RemoteSyncObject)
(events/publish-event! :event/field-update {:object field :user-id (mt/user->id :rasta)})
(is (empty? (t2/select :model/RemoteSyncObject))
- "no RSOs created — field has no user-curated metadata to track"))))
+ "no RSOs created — neither the Field nor the Table has user-curated metadata to track"))))
(deftest field-update-no-rso-for-unpublished-table-test
(testing "field-update creates no RSOs when the table is not published"
@@ -1301,23 +1297,24 @@
(events/publish-event! :event/field-update {:object field :user-id (mt/user->id :rasta)})
(is (empty? (t2/select :model/RemoteSyncObject))))))
-(deftest field-update-fus-rso-marks-removed-when-table-leaves-sync-scope-test
- (testing "existing FieldUserSettings RSO is marked removed when table leaves sync scope — no Field RSO created"
+(deftest field-update-table-user-settings-rso-marks-removed-when-table-leaves-sync-scope-test
+ (testing "existing TableUserSettings RSO is marked removed when the field's table leaves sync scope — no
+ Field or FieldUserSettings RSO created"
(mt/with-temp [:model/Collection synced {:is_remote_synced true :name "Synced" :type "library-data"}
:model/Collection normal {:name "Normal" :type "library-data"}
:model/Table table {:name "T" :is_published true :collection_id (:id synced)}
:model/Field field {:name "f" :table_id (:id table) :base_type :type/Text}]
(t2/insert! :model/FieldUserSettings {:field_id (:id field) :description "curated"})
(t2/delete! :model/RemoteSyncObject)
- (t2/insert! :model/RemoteSyncObject {:model_type "FieldUserSettings"
- :model_id (:id field)
- :model_name "f"
+ (t2/insert! :model/RemoteSyncObject {:model_type "TableUserSettings"
+ :model_id (:id table)
+ :model_name "T"
:model_table_id (:id table)
:status "synced"
:status_changed_at (t/offset-date-time)})
(t2/update! :model/Table (:id table) {:collection_id (:id normal)})
(events/publish-event! :event/field-update {:object field :user-id (mt/user->id :rasta)})
(let [entries (t2/select :model/RemoteSyncObject)]
- (is (= 1 (count entries)) "still only one RSO — no Field RSO added")
+ (is (= 1 (count entries)) "still only one RSO — no Field/FieldUserSettings RSO added")
(is (= "removed"
- (:status (t2/select-one :model/RemoteSyncObject :model_type "FieldUserSettings" :model_id (:id field)))))))))
+ (:status (t2/select-one :model/RemoteSyncObject :model_type "TableUserSettings" :model_id (:id table)))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
index ee994b5c666..0ca5f762544 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
@@ -2168,3 +2168,47 @@ serdes/meta:
:base-snapshot nil)]
(is (= :conflict (:status result)))
(is (str/includes? (:message result) "rewritten"))))))
+
+;;; --------------------------------- Table/Field user-settings inline round trip ---------------------------------
+
+(deftest table-and-field-user-settings-round-trip-test
+ (testing "a full export writes one file per published Table with user edits -- the TableUserSettings entity at
+ the Table's own path, inlining its edited Fields' settings under :fields -- and a later import
+ restores it, pruning field edits the file no longer lists"
+ (mt/with-temporary-setting-values [remote-sync-type :read-write]
+ (mt/with-temp [:model/Database {db-id :id} {:name "test-db"}
+ :model/Collection {coll-id :id} {:name "RS" :is_remote_synced true :location "/"}
+ :model/Table {table-id :id} {:name "Test Table" :db_id db-id
+ :is_published true :collection_id coll-id}
+ :model/Field {f1-id :id} {:name "F1" :table_id table-id :base_type :type/Text}
+ :model/Field {f2-id :id} {:name "F2" :table_id table-id :base_type :type/Text}]
+ (t2/insert! :model/TableUserSettings {:table_id table-id :display_name "Renamed"})
+ (t2/insert! :model/FieldUserSettings {:field_id f1-id :description "curated"})
+ (let [export-task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})
+ mock-source (test-helpers/create-mock-source)
+ export-result (impl/export! (source.p/snapshot mock-source) export-task-id "Test export" :force? true)]
+ (is (= :success (:status export-result)))
+ (remote-sync.task/complete-sync-task! export-task-id)
+ (let [files (get @(:files-atom mock-source) "main")
+ table-files (into {} (filter (fn [[p _]] (str/includes? p "test_table"))) files)]
+ (is (= 1 (count table-files))
+ (str "expected exactly one Table-related file, got " (keys table-files)))
+ (let [[path content] (first table-files)]
+ (is (str/ends-with? path "test_table.yaml")
+ "the file lives at the Table's own path, not a ___tableusersettings/___fieldusersettings file")
+ (is (not (str/includes? path "___tableusersettings")))
+ (is (not (str/includes? path "___fieldusersettings")))
+ (is (str/includes? content "Renamed"))
+ (is (str/includes? content "fields:"))
+ (is (str/includes? content "curated"))))
+ (t2/insert! :model/FieldUserSettings {:field_id f2-id :description "stale"})
+ (t2/update! :model/TableUserSettings :table_id table-id {:display_name "Local Edit"})
+ (let [import-task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
+ import-result (impl/import! (source.p/snapshot mock-source) import-task-id :force? true)]
+ (is (= :success (:status import-result)))
+ (is (= "Renamed" (:display_name (t2/select-one :model/TableUserSettings :table_id table-id)))
+ "T's user display_name is restored from the import")
+ (is (= "curated" (:description (t2/select-one :model/FieldUserSettings :field_id f1-id)))
+ "F1's edited description is present")
+ (is (not (t2/exists? :model/FieldUserSettings :field_id f2-id))
+ "F2's stale local row, absent from the imported file's :fields, is gone")))))))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj
index 920a76e9beb..36774016676 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj
@@ -623,3 +623,40 @@
"the warning covers only the unsynced subset (the potential data loss)")
(is (set/subset? flagged would-delete)
"everything the warning flags would indeed be removed")))))
+
+(deftest git-sync-exports-only-user-settings-test
+ (testing "git sync stores what users changed about a Table and its Fields, never the Table or Fields themselves --
+ those belong to sync, which runs against each instance's own warehouse -- as one TableUserSettings
+ entity per Table inlining its Fields' edits, never separate FieldUserSettings entities"
+ (mt/with-premium-features #{:library}
+ (mt/with-temp [:model/Collection {coll-id :id} {:is_remote_synced true :name "RS" :type "library-data"}
+ :model/Database {db-id :id} {:name "DB"}
+ :model/Table {table-id :id} {:name "T" :db_id db-id
+ :is_published true :collection_id coll-id}
+ :model/Field {f1 :id} {:name "F1" :table_id table-id}
+ :model/Field {f2 :id} {:name "F2" :table_id table-id}]
+ (t2/insert! :model/FieldUserSettings {:field_id f2 :description "curated" :description_set true})
+ (t2/insert! :model/TableUserSettings {:table_id table-id :display_name "Renamed"})
+ (let [exportable (spec/exportable-entities)]
+ (is (contains? (set (get exportable "TableUserSettings")) table-id)
+ "the Table's own edit, plus its edited Field's, are carried by one TableUserSettings entity")
+ (is (nil? (get exportable "FieldUserSettings"))
+ "FieldUserSettings is never exported as its own entity")
+ (is (not (contains? (set (get exportable "Table")) table-id)))
+ (is (empty? (filter #{f1 f2} (get exportable "Field")))))))))
+
+(deftest git-sync-exports-table-user-settings-for-field-only-edit-test
+ (testing "a Table with no TableUserSettings row of its own, but an edited Field, is still exportable -- the
+ TableUserSettings entity is synthesized to carry the Field's edit"
+ (mt/with-premium-features #{:library}
+ (mt/with-temp [:model/Collection {coll-id :id} {:is_remote_synced true :name "RS" :type "library-data"}
+ :model/Database {db-id :id} {:name "DB"}
+ :model/Table {table-id :id} {:name "T" :db_id db-id
+ :is_published true :collection_id coll-id}
+ :model/Field {f1 :id} {:name "F1" :table_id table-id}]
+ (t2/insert! :model/FieldUserSettings {:field_id f1 :description "curated" :description_set true})
+ (let [exportable (spec/exportable-entities)]
+ (is (not (t2/exists? :model/TableUserSettings :table_id table-id))
+ "the Table has no settings row of its own")
+ (is (contains? (set (get exportable "TableUserSettings")) table-id)
+ "the Table is still exportable, synthesized from its Field's edit"))))))
diff --git a/enterprise/backend/test/metabase_enterprise/replacement/runner_test.clj b/enterprise/backend/test/metabase_enterprise/replacement/runner_test.clj
index 72cc9edf982..75a891c0182 100644
--- a/enterprise/backend/test/metabase_enterprise/replacement/runner_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/replacement/runner_test.clj
@@ -266,7 +266,7 @@
[:stages 0 :source-card]))))))))))))))
(deftest copy-model-metadata-overrides!-test
- (testing "copies user-edited metadata from model result_metadata to Field and FieldUserSettings"
+ (testing "copies user-edited metadata from model result_metadata to FieldUserSettings only"
(mt/with-temp [:model/Table {table-id :id} {:name "transform_output"
:db_id (mt/id)
:active true}
@@ -299,14 +299,14 @@
:base_type "type/DateTimeWithLocalTZ"}])}
:where [:= :id card-id]})
(#'replacement.runner/copy-model-metadata-overrides! card-id table-id)
- (testing "Field records are updated with overrides from model metadata"
+ (testing "Field records are untouched; they stay sync-owned"
(let [field-1 (t2/select-one :model/Field :id field-1-id)
field-2 (t2/select-one :model/Field :id field-2-id)]
- (is (= "Order Total" (:display_name field-1)))
- (is (= "The total amount" (:description field-1)))
- (is (= :type/Currency (:semantic_type field-1)))
- (is (= "Order Date" (:display_name field-2)))
- (is (= :type/CreationTimestamp (:semantic_type field-2)))))
+ (is (= "Total" (:display_name field-1)))
+ (is (nil? (:description field-1)))
+ (is (nil? (:semantic_type field-1)))
+ (is (= "Created At" (:display_name field-2)))
+ (is (nil? (:semantic_type field-2)))))
(testing "FieldUserSettings are created so overrides survive sync"
(let [fus-1 (t2/select-one :model/FieldUserSettings :field_id field-1-id)
fus-2 (t2/select-one :model/FieldUserSettings :field_id field-2-id)]
@@ -344,8 +344,8 @@
:where [:= :id card-id]})
(#'replacement.runner/copy-model-metadata-overrides! card-id table-id)
(let [field (t2/select-one :model/Field :id field-id)]
- (is (= "Product ID" (:display_name field)))
- (is (= "The product identifier" (:description field))))
+ (is (= "Products ID" (:display_name field)))
+ (is (nil? (:description field))))
(let [fus (t2/select-one :model/FieldUserSettings :field_id field-id)]
(is (some? fus))
(is (= "Product ID" (:display_name fus)))
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj
index 1ad0a9ecc56..b72c17c817b 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj
@@ -24,6 +24,7 @@
:model/Field
:model/FieldValues
:model/FieldUserSettings
+ :model/TableUserSettings
;; OsiAiContext is identified by the entity it describes (entity_type + the entity's portable ref); its
;; serdes path nests under that entity, so it has no generated entity_id.
:model/OsiAiContext
@@ -164,7 +165,7 @@
(testing "All exported models should get entity id except those with other unique property (like name)"
(is (= (set (concat serdes.models/exported-models
;; those are inline models which still have entity_id
- ["DashboardCard" "DashboardTab" "Dimension" "MetabotPrompt"]))
+ ["DashboardCard" "DashboardTab" "Dimension" "MetabotPrompt" "FieldUserSettings"]))
(set (->> (concat entity-id-models
entities-external-name)
(map name))))))
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
index 4168d017321..b8530b6c9e4 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
@@ -1153,63 +1153,79 @@
:data_sensitivity :PII}
ser))
(is (not (contains? ser :field_id))
- ":field_id is dropped; its implied by the path")
- (testing "depend only on the Database; the parent Field is synthesized on import if missing"
- (is (= #{[{:model "Database" :id "My Database"}]}
- (set (serdes/deserialization-dependencies ser)))))))
+ ":field_id is dropped; its implied by the path")))
(testing "data_sensitivity on the Field itself"
(is (= :PII (:data_sensitivity (ts/extract-one "Field" field-id)))
"a labeled field exports the keyword as-is")
(is (not (contains? (ts/extract-one "Field" plain-id) :data_sensitivity))
"an unlabeled field exports no key, so nil never reaches the YAML"))
(testing "extract-metabase behavior"
- (let [models (->> {} (extract/extract) (map (comp :model last :serdes/meta)))]
- (is (= 1
- (t2/count :model/FieldUserSettings)
- (count (filter #{"FieldUserSettings"} models)))))))))
-
-(deftest table-descendants-user-edits-only-test
+ (let [entities (into [] (extract/extract {}))
+ models (map (comp :model last :serdes/meta) entities)]
+ (is (empty? (filter #{"FieldUserSettings"} models))
+ "written inside its Table's settings, never on its own")
+ (is (= 1 (count (filter #{"TableUserSettings"} models))))
+ (is (= 1 (count (:fields (first (filter #(= "TableUserSettings" (-> % :serdes/meta last :model)) entities)))))))))))
+
+(deftest table-descendants-user-settings-test
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc [:model/Database {db-id :id} {:name "DB"}
:model/Table {table-id :id} {:name "T" :db_id db-id}
:model/Field {f1-id :id} {:name "F1" :table_id table-id}
:model/Field {f2-id :id} {:name "F2" :table_id table-id}
:model/Field {f3-id :id} {:name "F3" :table_id table-id}]
- (testing "without user-edits-only: all fields returned as Field descendants"
- (let [desc (serdes/descendants "Table" table-id {})]
- (is (= #{["Field" f1-id] ["Field" f2-id] ["Field" f3-id]}
- (set (keys desc))))))
- (testing "with user-edits-only and no FieldUserSettings rows: no field descendants"
- (let [desc (serdes/descendants "Table" table-id {:user-edits-only true})]
- (is (empty? (filter (fn [[model _]] (#{"Field" "FieldUserSettings"} model)) (keys desc))))))
- (testing "with user-edits-only and one FieldUserSettings row: only that field appears as FieldUserSettings"
+ (testing "every Field is a descendant, and none has user settings to speak of yet"
+ (is (= #{["Field" f1-id] ["Field" f2-id] ["Field" f3-id]}
+ (set (keys (serdes/descendants "Table" table-id {}))))))
+ (testing "a Field with user settings makes the Table's settings a descendant, since they carry the Field's"
(t2/insert! :model/FieldUserSettings {:field_id f2-id :description "edited"})
- (let [desc (serdes/descendants "Table" table-id {:user-edits-only true})]
- (is (= #{["FieldUserSettings" f2-id]}
- (set (filter (fn [[model _]] (#{"Field" "FieldUserSettings"} model)) (keys desc))))))
+ (is (= #{["Field" f1-id] ["Field" f2-id] ["Field" f3-id] ["TableUserSettings" table-id]}
+ (set (keys (serdes/descendants "Table" table-id {})))))
(t2/delete! :model/FieldUserSettings :field_id f2-id))
- (testing "with user-edits-only and a FieldUserSettings row holding only data_sensitivity: that field appears as FieldUserSettings"
- (t2/insert! :model/FieldUserSettings {:field_id f1-id :data_sensitivity :PII})
- (let [desc (serdes/descendants "Table" table-id {:user-edits-only true})]
- (is (= #{["FieldUserSettings" f1-id]}
- (set (filter (fn [[model _]] (#{"Field" "FieldUserSettings"} model)) (keys desc))))))
- (t2/delete! :model/FieldUserSettings :field_id f1-id))
- (testing "with user-edits-only and all fields edited: all appear as FieldUserSettings, not Field"
- (t2/insert! :model/FieldUserSettings {:field_id f1-id})
- (t2/insert! :model/FieldUserSettings {:field_id f2-id})
- (t2/insert! :model/FieldUserSettings {:field_id f3-id})
- (let [desc (serdes/descendants "Table" table-id {:user-edits-only true})]
- (is (= #{["FieldUserSettings" f1-id] ["FieldUserSettings" f2-id] ["FieldUserSettings" f3-id]}
- (set (filter (fn [[model _]] (#{"Field" "FieldUserSettings"} model)) (keys desc)))))))
- (testing "Field and FieldUserSettings are leaf nodes in the descendants graph"
- ;; Table's descendants method is the only source of field-level entries; if Field ever
- ;; grows its own descendants (e.g. Field -> FieldUserSettings), traversal would visit
- ;; every field and full exports would change shape. Cement the leaf-ness here.
- (doseq [opts [{} {:user-edits-only true}]]
- (is (empty? (serdes/descendants "Field" f1-id opts)))
- (is (empty? (serdes/descendants "FieldUserSettings" f1-id opts))))))))
-
-(deftest user-edits-only-extract-test
+ (testing "a Table with user settings appears as TableUserSettings"
+ (t2/insert! :model/TableUserSettings {:table_id table-id :display_name "Renamed"})
+ (is (contains? (set (keys (serdes/descendants "Table" table-id {})))
+ ["TableUserSettings" table-id])))
+ (testing "Field is a leaf node in the descendants graph"
+ (is (empty? (serdes/descendants "Field" f1-id {})))))))
+
+(deftest inline-user-settings-extract-test
+ (mt/with-empty-h2-app-db!
+ (ts/with-temp-dpc [:model/Database {db-id :id} {:name "DB"}
+ :model/Table {table-id :id} {:name "T" :db_id db-id}
+ :model/Table {other-id :id} {:name "Unedited T" :db_id db-id}
+ :model/Field {f1-id :id} {:name "F1" :table_id table-id}
+ :model/Field {_f2-id :id} {:name "F2" :table_id table-id}
+ :model/FieldUserSettings _ {:field_id f1-id :description "edited"}]
+ (testing "a Table with only a Field edit synthesizes a TableUserSettings entity with that Field inlined"
+ (let [entities (into [] (serdes/extract-all "TableUserSettings"
+ {:inline-user-settings true
+ :filter-column :table_id
+ :filter-ids [table-id]}))]
+ (is (= 1 (count entities)))
+ (let [entity (first entities)]
+ (is (=? [{:model "Database" :id "DB"}
+ {:model "Table" :id "T"}
+ {:model "TableUserSettings" :id "1"}]
+ (:serdes/meta entity))
+ "synthesized for a Table with no settings row of its own")
+ (is (nil? (:display_name entity)))
+ (is (= 1 (count (:fields entity))))
+ (is (=? {:description "edited"
+ :serdes/meta [{:model "Database" :id "DB"}
+ {:model "Table" :id "T"}
+ {:model "Field" :id "F1"}
+ {:model "FieldUserSettings" :id "1"}]}
+ (first (:fields entity)))
+ "the edited Field is inlined, with its own serdes/meta")
+ (is (not (contains? (first (:fields entity)) :field_id))))))
+ (testing "a Table with no edits at all yields nothing"
+ (is (empty? (into [] (serdes/extract-all "TableUserSettings"
+ {:inline-user-settings true
+ :filter-column :table_id
+ :filter-ids [other-id]}))))))))
+
+(deftest collection-export-includes-user-settings-test
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc [:model/Database {db-id :id} {:name "DB"}
:model/Collection {coll-id :id} {:name "Library" :type "library-data"}
@@ -1218,24 +1234,19 @@
:collection_id coll-id}
:model/Field _ {:name "F1" :table_id table-id}
:model/Field {f2-id :id} {:name "F2" :table_id table-id}
- :model/FieldUserSettings _ {:field_id f2-id :description "curated"}]
- (testing "targeting the collection with user-edits-only: produces FieldUserSettings, not Field"
- (let [entities (into [] (extract/extract {:targets [["Collection" coll-id]]
- :user-edits-only true
- :no-data-model true}))
- by-model (group-by (comp :model last :serdes/meta) entities)]
- (is (contains? by-model "FieldUserSettings") "should include FieldUserSettings")
- (is (not (contains? by-model "Field")) "should not include Field")
- (is (= #{"F2"}
- (set (map #(-> % :serdes/meta (nth 2) :id) (by-model "FieldUserSettings"))))
- "only the edited field's FieldUserSettings (identified by field name in path)")
- (is (some #(= "T" (:name %)) (by-model "Table")) "the table itself is included")))
- (testing "without user-edits-only: produces Field, not FieldUserSettings"
+ :model/FieldUserSettings _ {:field_id f2-id :description "curated"}
+ :model/TableUserSettings _ {:table_id table-id :display_name "Renamed"}]
+ (testing "targeting the collection exports the Table and its Fields, each beside its user settings"
(let [entities (into [] (extract/extract {:targets [["Collection" coll-id]]
:no-data-model true}))
by-model (group-by (comp :model last :serdes/meta) entities)]
- (is (contains? by-model "Field") "should include Field")
- (is (not (contains? by-model "FieldUserSettings")) "should not include FieldUserSettings"))))))
+ (is (some #(= "T" (:name %)) (by-model "Table")))
+ (is (= #{"F1" "F2"}
+ (set (map :name (by-model "Field")))))
+ (is (= 1 (count (by-model "TableUserSettings"))))
+ (is (= #{"F2"}
+ (set (map #(-> % :serdes/meta (nth 2) :id) (:fields (first (by-model "TableUserSettings"))))))
+ "only the field the user edited has settings to export, inside the Table's"))))))
(deftest cards-test
(mt/with-empty-h2-app-db!
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
index 6b964bb4050..a6420418eae 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
@@ -142,6 +142,47 @@
(is (= "Basic Collection" (:name (first colls))))
(is (= eid1 (:entity_id (first colls)))))))))))
+(deftest inline-user-settings-round-trip-test
+ (testing "git sync's inline-user-settings mode replaces a Table's FieldUserSettings wholesale on import"
+ (let [serialized (atom nil)]
+ (ts/with-dbs [source-db dest-db]
+ (testing "extracting inline from the source"
+ (ts/with-db source-db
+ (let [db (ts/create! :model/Database :name "my-db")
+ table (ts/create! :model/Table :name "customers" :db_id (:id db))
+ f1 (ts/create! :model/Field :name "age" :table_id (:id table))
+ f2 (ts/create! :model/Field :name "email" :table_id (:id table))]
+ (t2/insert! :model/TableUserSettings {:table_id (:id table) :display_name "Renamed"})
+ (t2/insert! :model/FieldUserSettings {:field_id (:id f1) :description "edited"})
+ (reset! serialized
+ (into [(ts/extract-one "Database" (:id db))
+ (ts/extract-one "Table" (:id table))
+ (ts/extract-one "Field" (:id f1))
+ (ts/extract-one "Field" (:id f2))]
+ (serdes/extract-all "TableUserSettings"
+ {:inline-user-settings true
+ :filter-column :table_id
+ :filter-ids [(:id table)]}))))))
+ (testing "the extracted TableUserSettings inlines F1's edit and no others"
+ (let [tus (first (by-model @serialized "TableUserSettings"))]
+ (is (= "Renamed" (:display_name tus)))
+ (is (= 1 (count (:fields tus))))
+ (is (= "edited" (:description (first (:fields tus)))))))
+ (testing "loading into a destination where the same Database/Table/Fields exist, and F2 has a stale row"
+ (ts/with-db dest-db
+ (let [db (ts/create! :model/Database :name "my-db")
+ table (ts/create! :model/Table :name "customers" :db_id (:id db))
+ f1 (ts/create! :model/Field :name "age" :table_id (:id table))
+ f2 (ts/create! :model/Field :name "email" :table_id (:id table))]
+ (t2/insert! :model/FieldUserSettings {:field_id (:id f2) :description "stale"})
+ (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (testing "the Table's settings arrive"
+ (is (= "Renamed" (t2/select-one-fn :display_name :model/TableUserSettings :table_id (:id table)))))
+ (testing "F1's settings arrive"
+ (is (= "edited" (t2/select-one-fn :description :model/FieldUserSettings :field_id (:id f1)))))
+ (testing "F2's stale row is deleted -- replace semantics"
+ (is (nil? (t2/select-one :model/FieldUserSettings :field_id (:id f2))))))))))))
+
(deftest escape-continue-on-error-roundtrip-test
(testing "archive exported past escape analysis imports under continue-on-error without crashing (#74622)"
(let [serialized (atom nil)
@@ -2132,9 +2173,9 @@
(reset! serialized (into [] (serdes.extract/extract {})))
(testing "the labeled Field and its FieldUserSettings both export the label"
(is (= :PII (:data_sensitivity (field-ser @serialized "CONTACT_EMAIL"))))
- (is (= :PII (:data_sensitivity (u/seek #(and (in-db? %)
- (-> % :serdes/meta last :model (= "FieldUserSettings")))
- @serialized)))))
+ (is (= :PII (-> (u/seek #(and (in-db? %) (-> % :serdes/meta last :model (= "TableUserSettings")))
+ @serialized)
+ :fields first :data_sensitivity))))
(testing "the unlabeled Field exports no data_sensitivity key"
(is (not (contains? (field-ser @serialized "CONTACT_ROW") :data_sensitivity))))))
(ts/with-db dest-db
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/models_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/models_test.clj
index 128337d7eb9..d8f8930c6a0 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/models_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/models_test.clj
@@ -90,7 +90,8 @@
(testing "Nested models should declare `parent-ref`\n"
(doseq [[_nested transform] (filter #(::serdes/nested (second %)) spec')
:let [{:keys [model backward-fk]} transform
- inner-spec (serdes/make-spec (name model) nil)]]
+ inner-spec (serdes/make-spec (name model) nil)]
+ :when (not (:delete-children! (:opts transform)))]
(testing (format "%s has %s declared as `parent-ref`" model backward-fk)
(is (= (serdes/parent-ref)
(get-in inner-spec [:transform backward-fk]))))))
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj
index 5020d1ecd77..f46687f7250 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj
@@ -158,12 +158,25 @@
(storage/store! (into [] (extract/extract {})) (storage.files/file-writer dump-dir))
(let [fields-dir (io/file dump-dir "databases" "my_company_data" "tables" "customers" "fields")
read-yaml (fn [file-name] (yaml/from-file (io/file fields-dir file-name)))]
- (testing "the label is written as the bare enum string on both files"
+ (testing "the label is written as the bare enum string on the Field and inside the Table's settings"
(is (= "PII" (:data_sensitivity (read-yaml "email.yaml"))))
- (is (= "PII" (:data_sensitivity (read-yaml "email___fieldusersettings.yaml")))))
+ (is (= "PII" (-> (yaml/from-file (io/file fields-dir ".." "customers___tableusersettings.yaml"))
+ :fields first :data_sensitivity))))
(testing "an unlabeled field's file has no data_sensitivity line"
(is (not (contains? (read-yaml "id.yaml") :data_sensitivity)))))))))
+(deftest inline-user-settings-storage-path-test
+ (mt/with-empty-h2-app-db!
+ (ts/with-temp-dpc [:model/Database db {:name "My Company Data"}
+ :model/Table table {:name "Customers" :db_id (:id db)}
+ :model/Field email {:name "Email" :table_id (:id table)}
+ :model/FieldUserSettings _ {:field_id (:id email) :description "edited"}]
+ (let [[tus] (into [] (serdes/extract-all "TableUserSettings" {:filter-column :table_id :filter-ids [(:id table)]}))]
+ (is (= 1 (count (:fields tus))))
+ (testing "git sync writes the entity at the Table's own path"
+ (is (= "Customers" (-> (serdes/storage-path tus {:inline-user-settings true}) peek :label))))
+ (testing "the CLI export writes it beside the Table's file"
+ (is (= "Customers___tableusersettings" (-> (serdes/storage-path tus {}) peek :label))))))))
(deftest entity-counts-report-test
(ts/with-random-dump-dir [dump-dir "serdesv2-"]
(mt/with-empty-h2-app-db!
diff --git a/frontend/src/metabase/reference/reference.ts b/frontend/src/metabase/reference/reference.ts
index 920b87eb7fb..0d726bbfac5 100644
--- a/frontend/src/metabase/reference/reference.ts
+++ b/frontend/src/metabase/reference/reference.ts
@@ -67,8 +67,7 @@ const updateDataWrapper = (
try {
const editedFields = filterUntouchedFields(fields, props.entity);
if (!isEmptyObject(editedFields)) {
- const newEntity = { ...props.entity, ...editedFields };
- await fn(newEntity);
+ await fn({ id: props.entity.id, ...editedFields });
}
} finally {
resetForm(props);
@@ -127,7 +126,7 @@ export const rUpdateFields = (
),
}))
.filter(({ formField }) => !isEmptyObject(formField))
- .map(({ field, formField }) => ({ ...field, ...formField }));
+ .map(({ field, formField }) => ({ id: field.id, ...formField }));
await Promise.all(updatedFields.map(props.updateField));
} finally {
diff --git a/frontend/src/metabase/reference/update-actions.ts b/frontend/src/metabase/reference/update-actions.ts
index 4b7669af4b9..7f383b62b30 100644
--- a/frontend/src/metabase/reference/update-actions.ts
+++ b/frontend/src/metabase/reference/update-actions.ts
@@ -1,5 +1,3 @@
-import _ from "underscore";
-
import { databaseApi, fieldApi, segmentApi, tableApi } from "metabase/api";
import { runRtkEndpoint } from "metabase/api/utils/run-rtk-endpoint";
import {
@@ -8,19 +6,21 @@ import {
tableFetched,
} from "metabase/metadata-store";
import type { Dispatch } from "metabase/redux/store";
-import type { Database, Field, Segment, Table } from "metabase-types/api";
-
-/**
- * The edit forms in this module submit the entity they rendered, which came
- * from `state.entities` and carries the nested records the mirror stitched onto
- * it. The API rejects those, so each one is dropped before the request.
- */
+import type {
+ Database,
+ Field,
+ Table,
+ UpdateDatabaseRequest,
+ UpdateFieldRequest,
+ UpdateSegmentRequest,
+ UpdateTableRequest,
+} from "metabase-types/api";
export const updateDatabase =
- (database: Database) =>
+ (request: UpdateDatabaseRequest) =>
async (dispatch: Dispatch): Promise<unknown> => {
const updated: Database = await runRtkEndpoint(
- _.omit(database, "tables", "tables_lookup"),
+ request,
dispatch,
databaseApi.endpoints.updateDatabase,
);
@@ -29,16 +29,10 @@ export const updateDatabase =
};
export const updateTable =
- (table: Table) =>
+ (request: UpdateTableRequest) =>
async (dispatch: Dispatch): Promise<unknown> => {
const updated: Table = await runRtkEndpoint(
- _.omit(
- table,
- "fields",
- "fields_lookup",
- "aggregation_operators",
- "segments",
- ),
+ request,
dispatch,
tableApi.endpoints.updateTable,
);
@@ -47,10 +41,10 @@ export const updateTable =
};
export const updateField =
- (field: Field) =>
+ (request: UpdateFieldRequest) =>
async (dispatch: Dispatch): Promise<unknown> => {
const updated: Field = await runRtkEndpoint(
- _.omit(field, "filter_operators_lookup"),
+ request,
dispatch,
fieldApi.endpoints.updateField,
);
@@ -59,6 +53,6 @@ export const updateField =
};
export const updateSegment =
- (segment: Segment) =>
+ (request: UpdateSegmentRequest) =>
(dispatch: Dispatch): Promise<unknown> =>
- runRtkEndpoint(segment, dispatch, segmentApi.endpoints.updateSegment);
+ runRtkEndpoint(request, dispatch, segmentApi.endpoints.updateSegment);
diff --git a/mage/src/mage/modules.clj b/mage/src/mage/modules.clj
index 2dcd27274c3..f7fe38fa3dc 100644
--- a/mage/src/mage/modules.clj
+++ b/mage/src/mage/modules.clj
@@ -204,6 +204,7 @@
core.version
audit-app.view-log
warehouses.schema
+ warehouse-schema-overlay
xrays})
(defn- affected-modules
diff --git a/mage/src/mage/project_tests.clj b/mage/src/mage/project_tests.clj
index aabd05d866d..02c5bc89700 100644
--- a/mage/src/mage/project_tests.clj
+++ b/mage/src/mage/project_tests.clj
@@ -11,6 +11,9 @@
'[dev.modules-config-test
metabase.core.modules-test])
+(def ^:private honey-sql-check-namespaces
+ '[metabase.core.table-or-field-raw-usage-test])
+
(def ^:private ratchet-test-namespaces
;; These test the tooling itself. `./bin/mage kondo-ratchets` checks the source tree.
'[metabase.core.kondo-ratchet-test
@@ -20,7 +23,10 @@
'[metabase.core.namespace-uniqueness-test])
(def ^:private backend-check-namespaces
- (vec (concat module-check-namespaces ratchet-test-namespaces source-layout-check-namespaces)))
+ (vec (concat module-check-namespaces
+ ratchet-test-namespaces
+ source-layout-check-namespaces
+ honey-sql-check-namespaces)))
(def ^:private default-suites
"Suites the bare `project-tests` command runs, in order."
diff --git a/mage/test/mage/project_tests_test.clj b/mage/test/mage/project_tests_test.clj
index 473bf871511..202b94c3474 100644
--- a/mage/test/mage/project_tests_test.clj
+++ b/mage/test/mage/project_tests_test.clj
@@ -47,7 +47,8 @@
metabase.core.modules-test
metabase.core.kondo-ratchet-test
metabase.core.kondo-ratchet-check-test
- metabase.core.namespace-uniqueness-test])]]
+ metabase.core.namespace-uniqueness-test
+ metabase.core.table-or-field-raw-usage-test])]]
calls)))
(testing "the failed suite is reported"
(is (= ["migrations"] failed))
diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/db.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/db.clj
index 119fffecdd3..a82f03d1757 100644
--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/db.clj
+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/db.clj
@@ -4,6 +4,7 @@
(:require
[metabase.lib.schema.id :as lib.schema.id]
[metabase.util.malli :as mu]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[metabase.warehouses.schema :as warehouses.schema]
;; the driver persists dataset-filter and project-id migrations of its Database details back to the app DB
^{:clj-kondo/ignore [:discouraged-namespace]}
@@ -37,4 +38,5 @@
"Whether the Table with `table-id` has an active database-partitioned Field named `field-name`."
[table-id :- ::lib.schema.id/table
field-name :- :string]
- (t2/exists? :model/Field :table_id table-id :name field-name :database_partitioned true :active true))
+ (t2/exists? :model/Field :table_id table-id :name field-name :database_partitioned true :active true
+ {:from [(warehouse-schema-overlay/field-query {:user-settings? false})]}))
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/cache_creation_tokens.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/cache_creation_tokens.yaml
index ec44066cc37..79b23bed9e9 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/cache_creation_tokens.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/cache_creation_tokens.yaml
@@ -10,7 +10,6 @@ effective_type: type/Integer
semantic_type: type/Quantity
dimensions: []
position: 17
-custom_position: 0
database_position: 17
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/cache_read_tokens.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/cache_read_tokens.yaml
index 4ba8ddb0539..e72e8660efb 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/cache_read_tokens.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/cache_read_tokens.yaml
@@ -10,7 +10,6 @@ effective_type: type/Integer
semantic_type: type/Quantity
dimensions: []
position: 18
-custom_position: 0
database_position: 18
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/completion_tokens.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/completion_tokens.yaml
index 5e0d9113ea2..197def82744 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/completion_tokens.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/completion_tokens.yaml
@@ -10,7 +10,6 @@ effective_type: type/Integer
semantic_type: type/Quantity
dimensions: []
position: 7
-custom_position: 0
database_position: 7
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/conversation_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/conversation_id.yaml
index fa8427d61be..49c8b74e725 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/conversation_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/conversation_id.yaml
@@ -15,7 +15,6 @@ fk_target_field_id:
- conversation_id
dimensions: []
position: 9
-custom_position: 0
database_position: 9
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/created_at.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/created_at.yaml
index 1444b000956..f26fe527df3 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/created_at.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/created_at.yaml
@@ -13,7 +13,6 @@ effective_type: type/DateTimeWithLocalTZ
semantic_type: type/CreationTimestamp
dimensions: []
position: 1
-custom_position: 0
database_position: 1
serdes/meta:
- id: Internal Metabase Database
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/embedding_hostname.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/embedding_hostname.yaml
index b1c227e01c1..25cf49e3f23 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/embedding_hostname.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/embedding_hostname.yaml
@@ -10,7 +10,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 17
-custom_position: 0
database_position: 17
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/embedding_path.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/embedding_path.yaml
index 59c481c4672..bd88d7e8f58 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/embedding_path.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/embedding_path.yaml
@@ -9,7 +9,6 @@ base_type: type/Text
effective_type: type/Text
dimensions: []
position: 18
-custom_position: 0
database_position: 18
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/group_name.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/group_name.yaml
index 62b5a16cb65..360233f31db 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/group_name.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/group_name.yaml
@@ -10,7 +10,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 13
-custom_position: 0
database_position: 13
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/ip_address.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/ip_address.yaml
index a3a7f6b81f8..ddc5038adc2 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/ip_address.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/ip_address.yaml
@@ -9,7 +9,6 @@ base_type: type/Text
effective_type: type/Text
dimensions: []
position: 14
-custom_position: 0
database_position: 14
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/model.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/model.yaml
index dedee14ea46..0ff7bb92a61 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/model.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/model.yaml
@@ -10,7 +10,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 4
-custom_position: 0
database_position: 4
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/profile_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/profile_id.yaml
index 811bfa11027..d426cc0e98f 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/profile_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/profile_id.yaml
@@ -10,7 +10,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 5
-custom_position: 0
database_position: 5
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/prompt_tokens.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/prompt_tokens.yaml
index 0d382aefa28..9ee71f58401 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/prompt_tokens.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/prompt_tokens.yaml
@@ -10,7 +10,6 @@ effective_type: type/Integer
semantic_type: type/Quantity
dimensions: []
position: 6
-custom_position: 0
database_position: 6
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/request_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/request_id.yaml
index b6160c3c7d8..60afd950051 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/request_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/request_id.yaml
@@ -10,7 +10,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 16
-custom_position: 0
database_position: 16
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/sanitized_user_agent.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/sanitized_user_agent.yaml
index 4f768c4486d..4244d343033 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/sanitized_user_agent.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/sanitized_user_agent.yaml
@@ -10,7 +10,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 20
-custom_position: 0
database_position: 20
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/source.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/source.yaml
index 4c325d87750..849dd062df5 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/source.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/source.yaml
@@ -13,7 +13,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 2
-custom_position: 0
database_position: 2
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/source_name.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/source_name.yaml
index ba508822e1d..bcaca768ccb 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/source_name.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/source_name.yaml
@@ -10,7 +10,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 3
-custom_position: 0
database_position: 3
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/tenant_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/tenant_id.yaml
index c0d8537eee7..3b096bcf711 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/tenant_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/tenant_id.yaml
@@ -10,7 +10,6 @@ effective_type: type/Integer
semantic_type: type/FK
dimensions: []
position: 15
-custom_position: 0
database_position: 15
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/total_tokens.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/total_tokens.yaml
index 35748f68c7b..67be3cb3d35 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/total_tokens.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/total_tokens.yaml
@@ -10,7 +10,6 @@ effective_type: type/Integer
semantic_type: type/Quantity
dimensions: []
position: 8
-custom_position: 0
database_position: 8
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/usage_log_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/usage_log_id.yaml
index d76fb567695..188b8cb4cc2 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/usage_log_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/usage_log_id.yaml
@@ -13,7 +13,6 @@ effective_type: type/BigInteger
semantic_type: type/PK
dimensions: []
position: 0
-custom_position: 0
database_position: 0
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/user_agent.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/user_agent.yaml
index 0ee5b54b099..bdec95e1885 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/user_agent.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/user_agent.yaml
@@ -9,7 +9,6 @@ base_type: type/Text
effective_type: type/Text
dimensions: []
position: 19
-custom_position: 0
database_position: 19
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/user_display_name.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/user_display_name.yaml
index a353b48e17d..a1814ba18c4 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/user_display_name.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/user_display_name.yaml
@@ -10,7 +10,6 @@ effective_type: type/Text
semantic_type: type/Name
dimensions: []
position: 12
-custom_position: 0
database_position: 12
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/user_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/user_id.yaml
index 84317578e4d..83c0e2c8816 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/user_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/user_id.yaml
@@ -15,7 +15,6 @@ fk_target_field_id:
- user_id
dimensions: []
position: 10
-custom_position: 0
database_position: 10
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/user_qualified_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/user_qualified_id.yaml
index e71989d28f1..80833f40df2 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/user_qualified_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_ai_usage_log/fields/user_qualified_id.yaml
@@ -10,7 +10,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 11
-custom_position: 0
database_position: 11
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/alert_condition.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/alert_condition.yaml
index 801b6edc3f2..c059f4037a7 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/alert_condition.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/alert_condition.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 7
-custom_position: 0
database_position: 7
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/archived.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/archived.yaml
index e8d9da14c6c..8fc6e7cae39 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/archived.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/archived.yaml
@@ -9,7 +9,6 @@ effective_type: type/Boolean
semantic_type: type/Category
dimensions: []
position: 11
-custom_position: 0
database_position: 11
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/card_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/card_id.yaml
index fe1c3e4d882..eac676f3639 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/card_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/card_id.yaml
@@ -8,7 +8,6 @@ base_type: type/Integer
effective_type: type/Integer
dimensions: []
position: 5
-custom_position: 0
database_position: 5
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/card_qualified_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/card_qualified_id.yaml
index fb3f06a4fce..67bdc3dd393 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/card_qualified_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/card_qualified_id.yaml
@@ -17,7 +17,6 @@ dimensions:
serdes/meta:
- {id: nzL7XHp1yPu74r7BjcoJX, label: card_qualified_id, model: Dimension}
position: 6
-custom_position: 0
database_position: 6
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/created_at.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/created_at.yaml
index 792cf9c6ee4..5ad7f0e523a 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/created_at.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/created_at.yaml
@@ -9,7 +9,6 @@ effective_type: type/DateTimeWithLocalTZ
semantic_type: type/CreationTimestamp
dimensions: []
position: 2
-custom_position: 0
database_position: 2
serdes/meta:
- {id: Internal Metabase Database, model: Database}
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/creator_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/creator_id.yaml
index ea0514e9348..3b9a845dc36 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/creator_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/creator_id.yaml
@@ -17,7 +17,6 @@ dimensions:
serdes/meta:
- {id: AVykiWpDBL1s7oVrW86ru, label: creator_id, model: Dimension}
position: 4
-custom_position: 0
database_position: 4
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/dashboard_qualified_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/dashboard_qualified_id.yaml
index 355d6f8a514..05a7d2654c9 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/dashboard_qualified_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/dashboard_qualified_id.yaml
@@ -9,7 +9,6 @@ base_type: type/Text
effective_type: type/Text
dimensions: []
position: 12
-custom_position: 0
database_position: 12
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/entity_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/entity_id.yaml
index e9951d3afa0..f3c530d47ed 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/entity_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/entity_id.yaml
@@ -9,7 +9,6 @@ effective_type: type/Integer
semantic_type: type/PK
dimensions: []
position: 0
-custom_position: 0
database_position: 0
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/entity_qualified_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/entity_qualified_id.yaml
index 8ef6aeb570d..d93b2ddcd4c 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/entity_qualified_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/entity_qualified_id.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/PK
dimensions: []
position: 1
-custom_position: 0
database_position: 1
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/recipient_external.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/recipient_external.yaml
index 31da0fa9bb2..8dd4b8eefad 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/recipient_external.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/recipient_external.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 14
-custom_position: 0
database_position: 14
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/recipient_type.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/recipient_type.yaml
index 0be0a72db53..6a7fc74efb7 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/recipient_type.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/recipient_type.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 12
-custom_position: 0
database_position: 12
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/recipients.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/recipients.yaml
index f78952e60ea..4288c1e8d6e 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/recipients.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/recipients.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 13
-custom_position: 0
database_position: 13
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/schedule_day.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/schedule_day.yaml
index 9dfdb5e3a22..a0dd6376688 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/schedule_day.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/schedule_day.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 9
-custom_position: 0
database_position: 9
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/schedule_hour.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/schedule_hour.yaml
index 6fa085c2433..c0d1f897ab0 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/schedule_hour.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/schedule_hour.yaml
@@ -9,7 +9,6 @@ effective_type: type/Integer
semantic_type: type/Category
dimensions: []
position: 10
-custom_position: 0
database_position: 10
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/schedule_type.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/schedule_type.yaml
index e070f06c4b2..09f7e189e6c 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/schedule_type.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/schedule_type.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 8
-custom_position: 0
database_position: 8
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/updated_at.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/updated_at.yaml
index a99c5c8f895..0ff914444cc 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/updated_at.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_alerts/fields/updated_at.yaml
@@ -9,7 +9,6 @@ effective_type: type/DateTimeWithLocalTZ
semantic_type: type/UpdatedTimestamp
dimensions: []
position: 3
-custom_position: 0
database_position: 3
serdes/meta:
- {id: Internal Metabase Database, model: Database}
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_audit_log/fields/details.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_audit_log/fields/details.yaml
index a86f0d89fee..d4d59534ec7 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_audit_log/fields/details.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_audit_log/fields/details.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 8
-custom_position: 0
database_position: 8
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_audit_log/fields/end_timestamp.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_audit_log/fields/end_timestamp.yaml
index 772a7e12fb2..4ec356ef341 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_audit_log/fields/end_timestamp.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_audit_log/fields/end_timestamp.yaml
@@ -8,7 +8,6 @@ base_type: type/Text
effective_type: type/Text
dimensions: []
position: 3
-custom_position: 0
database_position: 3
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_audit_log/fields/entity_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_audit_log/fields/entity_id.yaml
index 9d06cd9f4d6..8976a53c222 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_audit_log/fields/entity_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_audit_log/fields/entity_id.yaml
@@ -9,7 +9,6 @@ effective_type: type/Integer
semantic_type: type/Category
dimensions: []
position: 6
-custom_position: 0
database_position: 6
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_audit_log/fields/entity_qualified_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_audit_log/fields/entity_qualified_id.yaml
index 9f6f85aa1ec..96d7cd3eaa5 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_audit_log/fields/entity_qualified_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_audit_log/fields/entity_qualified_id.yaml
@@ -17,7 +17,6 @@ dimensions:
serdes/meta:
- {id: 3AntG1-Ylk_c9KthNvyxV, label: entity_qualified_id, model: Dimension}
position: 7
-custom_position: 0
database_position: 7
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_audit_log/fields/entity_type.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_audit_log/fields/entity_type.yaml
index c4f6ef58b32..c003c8162b9 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_audit_log/fields/entity_type.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_audit_log/fields/entity_type.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 5
-custom_position: 0
database_position: 5
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_audit_log/fields/id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_audit_log/fields/id.yaml
index 0cbdb648202..c8f4a7b115a 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_audit_log/fields/id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_audit_log/fields/id.yaml
@@ -9,7 +9,6 @@ effective_type: type/Integer
semantic_type: type/PK
dimensions: []
position: 0
-custom_position: 0
database_position: 0
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_audit_log/fields/model.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_audit_log/fields/model.yaml
index 9bbf402a785..69b2caa5a06 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_audit_log/fields/model.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_audit_log/fields/model.yaml
@@ -10,7 +10,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 5
-custom_position: 0
database_position: 5
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_audit_log/fields/model_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_audit_log/fields/model_id.yaml
index 531d1ebedd1..da50e465dab 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_audit_log/fields/model_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_audit_log/fields/model_id.yaml
@@ -11,7 +11,6 @@ semantic_type: type/FK
fk_target_field_id: [Internal Metabase Database, public, v_content, id]
dimensions: []
position: 6
-custom_position: 0
database_position: 6
serdes/meta:
- {id: Internal Metabase Database, model: Database}
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_audit_log/fields/timestamp.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_audit_log/fields/timestamp.yaml
index 802b4657f84..11f7ff8dc74 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_audit_log/fields/timestamp.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_audit_log/fields/timestamp.yaml
@@ -9,7 +9,6 @@ effective_type: type/DateTimeWithLocalTZ
semantic_type: type/CreationTimestamp
dimensions: []
position: 2
-custom_position: 0
database_position: 2
serdes/meta:
- {id: Internal Metabase Database, model: Database}
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_audit_log/fields/topic.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_audit_log/fields/topic.yaml
index 2b4d0506ae6..cba31cad4ef 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_audit_log/fields/topic.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_audit_log/fields/topic.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 1
-custom_position: 0
database_position: 1
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_audit_log/fields/user_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_audit_log/fields/user_id.yaml
index 1d13da20ef2..1df02bd715d 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_audit_log/fields/user_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_audit_log/fields/user_id.yaml
@@ -17,7 +17,6 @@ dimensions:
serdes/meta:
- {id: pvLruX9Qkzkb9cLg9eW5J, label: user_id, model: Dimension}
position: 4
-custom_position: 0
database_position: 4
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/action_model_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/action_model_id.yaml
index 506cfbe5d23..ea364deff4a 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/action_model_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/action_model_id.yaml
@@ -8,7 +8,6 @@ base_type: type/Integer
effective_type: type/Integer
dimensions: []
position: 14
-custom_position: 0
database_position: 14
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/action_type.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/action_type.yaml
index 676cb726256..98d34108824 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/action_type.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/action_type.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 13
-custom_position: 0
database_position: 13
has_field_values: list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/archived.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/archived.yaml
index c319cdbce52..26991b6e02f 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/archived.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/archived.yaml
@@ -9,7 +9,6 @@ effective_type: type/Boolean
semantic_type: type/Enum
dimensions: []
position: 12
-custom_position: 0
database_position: 12
has_field_values: list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/collection_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/collection_id.yaml
index 0dc8d15edd9..8e8236aea80 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/collection_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/collection_id.yaml
@@ -8,7 +8,6 @@ base_type: type/Integer
effective_type: type/Integer
dimensions: []
position: 8
-custom_position: 0
database_position: 8
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/collection_is_official.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/collection_is_official.yaml
index 9b505be1dc8..bf71d754bba 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/collection_is_official.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/collection_is_official.yaml
@@ -9,7 +9,6 @@ effective_type: type/Boolean
semantic_type: type/Category
dimensions: []
position: 15
-custom_position: 0
database_position: 15
has_field_values: list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/collection_is_personal.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/collection_is_personal.yaml
index 7a9afd188ec..77656a6ded8 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/collection_is_personal.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/collection_is_personal.yaml
@@ -9,7 +9,6 @@ effective_type: type/Boolean
semantic_type: type/Category
dimensions: []
position: 16
-custom_position: 0
database_position: 16
has_field_values: list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/created_at.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/created_at.yaml
index 1ff61a8bdff..5209f0bbc4c 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/created_at.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/created_at.yaml
@@ -9,7 +9,6 @@ effective_type: type/DateTimeWithLocalTZ
semantic_type: type/CreationTimestamp
dimensions: []
position: 3
-custom_position: 0
database_position: 3
serdes/meta:
- {id: Internal Metabase Database, model: Database}
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/creator_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/creator_id.yaml
index 26c3520ac4c..1df28f3b3dc 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/creator_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/creator_id.yaml
@@ -17,7 +17,6 @@ dimensions:
serdes/meta:
- {id: pSDZP-3_Kb-tT053QKI2C, label: creator_id, model: Dimension}
position: 5
-custom_position: 0
database_position: 5
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/description.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/description.yaml
index 5250a3ce2c5..ccba5832a17 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/description.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/description.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Description
dimensions: []
position: 7
-custom_position: 0
database_position: 7
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/entity_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/entity_id.yaml
index 4ba35e69abe..ed5cc08ff40 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/entity_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/entity_id.yaml
@@ -9,7 +9,6 @@ effective_type: type/Integer
semantic_type: type/PK
dimensions: []
position: 0
-custom_position: 0
database_position: 0
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/entity_qualified_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/entity_qualified_id.yaml
index 3a29377762f..e830b671de1 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/entity_qualified_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/entity_qualified_id.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/PK
dimensions: []
position: 1
-custom_position: 0
database_position: 1
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/entity_type.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/entity_type.yaml
index 030c7c8071c..3be80fa7df7 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/entity_type.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/entity_type.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 2
-custom_position: 0
database_position: 2
has_field_values: list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/event_timestamp.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/event_timestamp.yaml
index fcdb7de80c1..723a53aaa6b 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/event_timestamp.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/event_timestamp.yaml
@@ -8,7 +8,6 @@ base_type: type/DateTimeWithLocalTZ
effective_type: type/DateTimeWithLocalTZ
dimensions: []
position: 20
-custom_position: 0
database_position: 20
serdes/meta:
- {id: Internal Metabase Database, model: Database}
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/id.yaml
index 8b609930038..2fc231e8e19 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/id.yaml
@@ -10,7 +10,6 @@ effective_type: type/Text
semantic_type: type/PK
dimensions: []
position: 0
-custom_position: 0
database_position: 0
serdes/meta:
- {id: Internal Metabase Database, model: Database}
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/is_embedding_enabled.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/is_embedding_enabled.yaml
index f5ed48e01ca..4d6ee01237b 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/is_embedding_enabled.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/is_embedding_enabled.yaml
@@ -9,7 +9,6 @@ effective_type: type/Boolean
semantic_type: type/Category
dimensions: []
position: 10
-custom_position: 0
database_position: 10
has_field_values: list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/is_verified.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/is_verified.yaml
index ae4b5d71605..3c9c1521cd9 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/is_verified.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/is_verified.yaml
@@ -9,7 +9,6 @@ effective_type: type/Boolean
semantic_type: type/Category
dimensions: []
position: 11
-custom_position: 0
database_position: 11
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/made_public_by_user.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/made_public_by_user.yaml
index af9fce001ef..4189c5c2ad5 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/made_public_by_user.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/made_public_by_user.yaml
@@ -10,7 +10,6 @@ semantic_type: type/FK
fk_target_field_id: [Internal Metabase Database, public, v_users, user_id]
dimensions: []
position: 9
-custom_position: 0
database_position: 9
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/name.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/name.yaml
index 312f8b4bb45..56d3f432315 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/name.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/name.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Name
dimensions: []
position: 6
-custom_position: 0
database_position: 6
has_field_values: list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/question_database_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/question_database_id.yaml
index 36bcb006994..c497a87c19d 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/question_database_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/question_database_id.yaml
@@ -10,7 +10,6 @@ semantic_type: type/FK
fk_target_field_id: [Internal Metabase Database, public, v_databases, entity_qualified_id]
dimensions: []
position: 18
-custom_position: 0
database_position: 18
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/question_is_native.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/question_is_native.yaml
index 08f8e822872..e5fa1d8c91c 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/question_is_native.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/question_is_native.yaml
@@ -9,7 +9,6 @@ effective_type: type/Boolean
semantic_type: type/Category
dimensions: []
position: 19
-custom_position: 0
database_position: 19
has_field_values: list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/question_viz_type.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/question_viz_type.yaml
index 3bd8ccf77f4..c6662d66074 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/question_viz_type.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/question_viz_type.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 17
-custom_position: 0
database_position: 17
has_field_values: list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/updated_at.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/updated_at.yaml
index f707a87f27f..0c48d60a490 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/updated_at.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_content/fields/updated_at.yaml
@@ -9,7 +9,6 @@ effective_type: type/DateTimeWithLocalTZ
semantic_type: type/UpdatedTimestamp
dimensions: []
position: 4
-custom_position: 0
database_position: 4
serdes/meta:
- {id: Internal Metabase Database, model: Database}
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/card_qualified_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/card_qualified_id.yaml
index e2ee3a22d67..4bc0cc56eaa 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/card_qualified_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/card_qualified_id.yaml
@@ -17,7 +17,6 @@ dimensions:
serdes/meta:
- {id: YxACoUnvO-zVZNnnUZhr-, label: card_qualified_id, model: Dimension}
position: 4
-custom_position: 0
database_position: 4
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/col.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/col.yaml
index 6aa5958ad4a..7cc7555c17e 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/col.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/col.yaml
@@ -10,7 +10,6 @@ effective_type: type/Integer
semantic_type: type/Category
dimensions: []
position: 9
-custom_position: 0
database_position: 9
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/created_at.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/created_at.yaml
index a5d096cdb29..469045be358 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/created_at.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/created_at.yaml
@@ -9,7 +9,6 @@ effective_type: type/DateTimeWithLocalTZ
semantic_type: type/CreationTimestamp
dimensions: []
position: 5
-custom_position: 0
database_position: 5
serdes/meta:
- {id: Internal Metabase Database, model: Database}
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/dashboard_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/dashboard_id.yaml
index b93e92f4db1..90c5c94378f 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/dashboard_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/dashboard_id.yaml
@@ -9,7 +9,6 @@ base_type: type/Text
effective_type: type/Text
dimensions: []
position: 2
-custom_position: 0
database_position: 2
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/dashboard_qualified_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/dashboard_qualified_id.yaml
index effdea4aacc..171b827d073 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/dashboard_qualified_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/dashboard_qualified_id.yaml
@@ -17,7 +17,6 @@ dimensions:
serdes/meta:
- {id: wF4R7X-pfOv-fgODquyKR, label: dashboard_qualified_id, model: Dimension}
position: 2
-custom_position: 0
database_position: 2
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/dashboardtab_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/dashboardtab_id.yaml
index 429453d88e3..3cf77626115 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/dashboardtab_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/dashboardtab_id.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 3
-custom_position: 0
database_position: 3
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/entity_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/entity_id.yaml
index e834990233b..793365912e8 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/entity_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/entity_id.yaml
@@ -9,7 +9,6 @@ effective_type: type/Integer
semantic_type: type/PK
dimensions: []
position: 0
-custom_position: 0
database_position: 0
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/entity_qualified_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/entity_qualified_id.yaml
index bd62cc3480e..0da0c1d56e9 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/entity_qualified_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/entity_qualified_id.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/PK
dimensions: []
position: 1
-custom_position: 0
database_position: 1
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/id.yaml
index dc8452ff2b7..b2c7ae057d1 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/id.yaml
@@ -10,7 +10,6 @@ effective_type: type/Text
semantic_type: type/PK
dimensions: []
position: 0
-custom_position: 0
database_position: 0
serdes/meta:
- {id: Internal Metabase Database, model: Database}
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/parameter_mappings.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/parameter_mappings.yaml
index 66f7bf2fe78..397d4cb9ffc 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/parameter_mappings.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/parameter_mappings.yaml
@@ -10,7 +10,6 @@ semantic_type: type/SerializedJSON
dimensions: []
preview_display: false
position: 10
-custom_position: 0
database_position: 10
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/question_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/question_id.yaml
index 897b9399685..f09ffae0c3e 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/question_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/question_id.yaml
@@ -9,7 +9,6 @@ base_type: type/Text
effective_type: type/Text
dimensions: []
position: 4
-custom_position: 0
database_position: 4
serdes/meta:
- {id: Internal Metabase Database, model: Database}
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/row.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/row.yaml
index c2c92ea676c..06d74c621fd 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/row.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/row.yaml
@@ -9,7 +9,6 @@ base_type: type/Integer
effective_type: type/Integer
dimensions: []
position: 8
-custom_position: 0
database_position: 8
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/size_x.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/size_x.yaml
index aae693050ba..e635d488096 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/size_x.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/size_x.yaml
@@ -8,7 +8,6 @@ base_type: type/Integer
effective_type: type/Integer
dimensions: []
position: 7
-custom_position: 0
database_position: 7
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/size_y.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/size_y.yaml
index f3ad87d7dbf..a85f87d840f 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/size_y.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/size_y.yaml
@@ -8,7 +8,6 @@ base_type: type/Integer
effective_type: type/Integer
dimensions: []
position: 8
-custom_position: 0
database_position: 8
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/updated_at.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/updated_at.yaml
index d4f23e81b2a..48c279b9308 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/updated_at.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/updated_at.yaml
@@ -9,7 +9,6 @@ effective_type: type/DateTimeWithLocalTZ
semantic_type: type/UpdatedTimestamp
dimensions: []
position: 6
-custom_position: 0
database_position: 6
serdes/meta:
- {id: Internal Metabase Database, model: Database}
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/visualization_settings.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/visualization_settings.yaml
index 235568d3a13..8de06d27874 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/visualization_settings.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_dashboardcard/fields/visualization_settings.yaml
@@ -10,7 +10,6 @@ semantic_type: type/SerializedJSON
dimensions: []
preview_display: false
position: 9
-custom_position: 0
database_position: 9
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/auto_run_queries.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/auto_run_queries.yaml
index 845d0a4b367..9ff81c770a8 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/auto_run_queries.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/auto_run_queries.yaml
@@ -9,7 +9,6 @@ effective_type: type/Boolean
semantic_type: type/Category
dimensions: []
position: 11
-custom_position: 0
database_position: 11
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/cache_field_values_schedule.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/cache_field_values_schedule.yaml
index cc19207f12d..da50fa57c6d 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/cache_field_values_schedule.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/cache_field_values_schedule.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 8
-custom_position: 0
database_position: 8
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/cache_ttl.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/cache_ttl.yaml
index 338e692bac9..37ee5794604 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/cache_ttl.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/cache_ttl.yaml
@@ -8,7 +8,6 @@ base_type: type/Integer
effective_type: type/Integer
dimensions: []
position: 12
-custom_position: 0
database_position: 12
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/created_at.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/created_at.yaml
index c4bb431afac..80691a01b36 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/created_at.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/created_at.yaml
@@ -9,7 +9,6 @@ effective_type: type/DateTimeWithLocalTZ
semantic_type: type/CreationTimestamp
dimensions: []
position: 2
-custom_position: 0
database_position: 2
serdes/meta:
- {id: Internal Metabase Database, model: Database}
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/creator_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/creator_id.yaml
index e5a581051e0..9528fc5d624 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/creator_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/creator_id.yaml
@@ -17,7 +17,6 @@ dimensions:
serdes/meta:
- {id: JWvkJb39WwSrgKTtJ9JWM, label: creator_id, model: Dimension}
position: 13
-custom_position: 0
database_position: 13
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/database_type.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/database_type.yaml
index 2e12e69a91b..81b028e243d 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/database_type.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/database_type.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 6
-custom_position: 0
database_position: 6
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/db_version.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/db_version.yaml
index f1ac281e8c6..6f6651d77dc 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/db_version.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/db_version.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 14
-custom_position: 0
database_position: 14
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/description.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/description.yaml
index 09714e9e92e..2cb5a4107ff 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/description.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/description.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Description
dimensions: []
position: 5
-custom_position: 0
database_position: 5
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/entity_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/entity_id.yaml
index 4e401326460..e5712e67617 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/entity_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/entity_id.yaml
@@ -9,7 +9,6 @@ effective_type: type/Integer
semantic_type: type/PK
dimensions: []
position: 0
-custom_position: 0
database_position: 0
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/entity_qualified_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/entity_qualified_id.yaml
index 54296b33d6f..ba97dacdb3c 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/entity_qualified_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/entity_qualified_id.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/PK
dimensions: []
position: 1
-custom_position: 0
database_position: 1
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/is_on_demand.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/is_on_demand.yaml
index b1ed642cf8a..c542399d476 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/is_on_demand.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/is_on_demand.yaml
@@ -9,7 +9,6 @@ effective_type: type/Boolean
semantic_type: type/Category
dimensions: []
position: 10
-custom_position: 0
database_position: 10
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/metadata_sync_schedule.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/metadata_sync_schedule.yaml
index c6fac8101d4..d3b98860f70 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/metadata_sync_schedule.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/metadata_sync_schedule.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 7
-custom_position: 0
database_position: 7
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/name.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/name.yaml
index f26c9808371..c5dba4d07a0 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/name.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/name.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Name
dimensions: []
position: 4
-custom_position: 0
database_position: 4
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/timezone.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/timezone.yaml
index bea337fdd5f..d3e91551144 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/timezone.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/timezone.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 9
-custom_position: 0
database_position: 9
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/updated_at.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/updated_at.yaml
index a4e1449dc05..218789c3057 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/updated_at.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_databases/fields/updated_at.yaml
@@ -9,7 +9,6 @@ effective_type: type/DateTimeWithLocalTZ
semantic_type: type/UpdatedTimestamp
dimensions: []
position: 3
-custom_position: 0
database_position: 3
serdes/meta:
- {id: Internal Metabase Database, model: Database}
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/active.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/active.yaml
index 691134c094b..9a44cc3e67d 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/active.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/active.yaml
@@ -9,7 +9,6 @@ effective_type: type/Boolean
semantic_type: type/Category
dimensions: []
position: 11
-custom_position: 0
database_position: 11
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/base_type.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/base_type.yaml
index 76cfd935834..cde344291fa 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/base_type.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/base_type.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 7
-custom_position: 0
database_position: 7
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/created_at.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/created_at.yaml
index 9a36664492d..236f07461b4 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/created_at.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/created_at.yaml
@@ -9,7 +9,6 @@ effective_type: type/DateTimeWithLocalTZ
semantic_type: type/CreationTimestamp
dimensions: []
position: 2
-custom_position: 0
database_position: 2
serdes/meta:
- {id: Internal Metabase Database, model: Database}
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/description.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/description.yaml
index 5ce9a68c61e..e572ca888f2 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/description.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/description.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Description
dimensions: []
position: 6
-custom_position: 0
database_position: 6
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/display_name.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/display_name.yaml
index b5f2696c23e..46c2d8fbfc3 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/display_name.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/display_name.yaml
@@ -8,7 +8,6 @@ base_type: type/Text
effective_type: type/Text
dimensions: []
position: 5
-custom_position: 0
database_position: 5
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/entity_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/entity_id.yaml
index 478a120d6d4..d4a2a4cc9f3 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/entity_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/entity_id.yaml
@@ -9,7 +9,6 @@ effective_type: type/Integer
semantic_type: type/PK
dimensions: []
position: 0
-custom_position: 0
database_position: 0
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/entity_qualified_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/entity_qualified_id.yaml
index f92fe0aa355..cc56aad1076 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/entity_qualified_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/entity_qualified_id.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/PK
dimensions: []
position: 1
-custom_position: 0
database_position: 1
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/fk_target_field_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/fk_target_field_id.yaml
index 44c587c7c99..23423f9dfdd 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/fk_target_field_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/fk_target_field_id.yaml
@@ -17,7 +17,6 @@ dimensions:
serdes/meta:
- {id: Dl90EVZMrh8F4vog-lzt1, label: fk_target_field_id, model: Dimension}
position: 9
-custom_position: 0
database_position: 9
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/has_field_values.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/has_field_values.yaml
index 02eba760368..6076d28c6ab 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/has_field_values.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/has_field_values.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 10
-custom_position: 0
database_position: 10
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/name.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/name.yaml
index c897c4c7313..303803c2814 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/name.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/name.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Name
dimensions: []
position: 4
-custom_position: 0
database_position: 4
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/table_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/table_id.yaml
index 0dc4549d6cd..6f75dd4ea27 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/table_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/table_id.yaml
@@ -17,7 +17,6 @@ dimensions:
serdes/meta:
- {id: ymOoU4Ho_xLPmyJbFdaS7, label: table_id, model: Dimension}
position: 12
-custom_position: 0
database_position: 12
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/updated_at.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/updated_at.yaml
index c170f126310..ddaa146973c 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/updated_at.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/updated_at.yaml
@@ -9,7 +9,6 @@ effective_type: type/DateTimeWithLocalTZ
semantic_type: type/UpdatedTimestamp
dimensions: []
position: 3
-custom_position: 0
database_position: 3
serdes/meta:
- {id: Internal Metabase Database, model: Database}
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/visibility_type.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/visibility_type.yaml
index c27f76b0680..367a164dcec 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/visibility_type.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_fields/fields/visibility_type.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 8
-custom_position: 0
database_position: 8
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_group_members/fields/group_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_group_members/fields/group_id.yaml
index a7066b6e671..75036d77a32 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_group_members/fields/group_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_group_members/fields/group_id.yaml
@@ -9,7 +9,6 @@ effective_type: type/Integer
semantic_type: type/PK
dimensions: []
position: 1
-custom_position: 0
database_position: 1
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_group_members/fields/group_name.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_group_members/fields/group_name.yaml
index 59389e7209e..e71054ccb93 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_group_members/fields/group_name.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_group_members/fields/group_name.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Name
dimensions: []
position: 2
-custom_position: 0
database_position: 2
has_field_values: list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_group_members/fields/user_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_group_members/fields/user_id.yaml
index d16eefc7310..02436a295a7 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_group_members/fields/user_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_group_members/fields/user_id.yaml
@@ -9,7 +9,6 @@ effective_type: type/Integer
semantic_type: type/Description
dimensions: []
position: 0
-custom_position: 0
database_position: 0
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/assistant_message_count.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/assistant_message_count.yaml
index cba261c34f4..c0a546bbd4c 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/assistant_message_count.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/assistant_message_count.yaml
@@ -13,7 +13,6 @@ effective_type: type/BigInteger
semantic_type: type/Quantity
dimensions: []
position: 7
-custom_position: 0
database_position: 7
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/completion_tokens.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/completion_tokens.yaml
index 9ccd97fcf97..2153570d443 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/completion_tokens.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/completion_tokens.yaml
@@ -13,7 +13,6 @@ effective_type: type/BigInteger
semantic_type: type/Quantity
dimensions: []
position: 10
-custom_position: 0
database_position: 10
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/conversation_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/conversation_id.yaml
index 224f9241537..1821c014623 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/conversation_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/conversation_id.yaml
@@ -13,7 +13,6 @@ effective_type: type/Text
semantic_type: type/PK
dimensions: []
position: 0
-custom_position: 0
database_position: 0
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/created_at.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/created_at.yaml
index a4e374093c2..7e992f6b32f 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/created_at.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/created_at.yaml
@@ -13,7 +13,6 @@ effective_type: type/DateTimeWithLocalTZ
semantic_type: type/CreationTimestamp
dimensions: []
position: 1
-custom_position: 0
database_position: 1
serdes/meta:
- id: Internal Metabase Database
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/embedding_hostname.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/embedding_hostname.yaml
index 3fb66c2bc45..0139cd72548 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/embedding_hostname.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/embedding_hostname.yaml
@@ -10,7 +10,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 21
-custom_position: 0
database_position: 21
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/embedding_path.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/embedding_path.yaml
index 1fe87f46775..5cac13f2ac8 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/embedding_path.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/embedding_path.yaml
@@ -9,7 +9,6 @@ base_type: type/Text
effective_type: type/Text
dimensions: []
position: 22
-custom_position: 0
database_position: 22
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/group_name.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/group_name.yaml
index fa202b6a9d2..ea9186ad211 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/group_name.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/group_name.yaml
@@ -10,7 +10,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 14
-custom_position: 0
database_position: 14
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/ip_address.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/ip_address.yaml
index 0cf3a7408a0..5e428bd922a 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/ip_address.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/ip_address.yaml
@@ -9,7 +9,6 @@ base_type: type/Text
effective_type: type/Text
dimensions: []
position: 17
-custom_position: 0
database_position: 17
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/last_message_at.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/last_message_at.yaml
index 08bef4ece8a..f522d2d75a7 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/last_message_at.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/last_message_at.yaml
@@ -12,7 +12,6 @@ base_type: type/DateTimeWithLocalTZ
effective_type: type/DateTimeWithLocalTZ
dimensions: []
position: 11
-custom_position: 0
database_position: 11
serdes/meta:
- id: Internal Metabase Database
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/message_count.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/message_count.yaml
index e7c286253cc..78ba59b96de 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/message_count.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/message_count.yaml
@@ -13,7 +13,6 @@ effective_type: type/BigInteger
semantic_type: type/Quantity
dimensions: []
position: 5
-custom_position: 0
database_position: 5
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/model.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/model.yaml
index 7cb976fd5ca..fa4af7f8c45 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/model.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/model.yaml
@@ -10,7 +10,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 20
-custom_position: 0
database_position: 20
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/profile_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/profile_id.yaml
index 680bd9c75cc..8e005aef699 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/profile_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/profile_id.yaml
@@ -13,7 +13,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 12
-custom_position: 0
database_position: 12
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/profile_name.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/profile_name.yaml
index 22fd64cc8d9..c7a5afa4db8 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/profile_name.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/profile_name.yaml
@@ -10,7 +10,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 13
-custom_position: 0
database_position: 13
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/prompt_tokens.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/prompt_tokens.yaml
index 797dfe3b614..eb021200e1c 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/prompt_tokens.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/prompt_tokens.yaml
@@ -13,7 +13,6 @@ effective_type: type/BigInteger
semantic_type: type/Quantity
dimensions: []
position: 9
-custom_position: 0
database_position: 9
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/sanitized_user_agent.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/sanitized_user_agent.yaml
index b400cf36f1e..14232e210a0 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/sanitized_user_agent.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/sanitized_user_agent.yaml
@@ -10,7 +10,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 24
-custom_position: 0
database_position: 24
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/source.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/source.yaml
index 99d1a0115fa..60904dc6a60 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/source.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/source.yaml
@@ -10,7 +10,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 15
-custom_position: 0
database_position: 15
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/source_name.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/source_name.yaml
index 7569f7e1250..46e255e3a02 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/source_name.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/source_name.yaml
@@ -10,7 +10,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 16
-custom_position: 0
database_position: 16
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/tenant_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/tenant_id.yaml
index 891e17aaf4e..9f45e03b600 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/tenant_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/tenant_id.yaml
@@ -10,7 +10,6 @@ effective_type: type/Integer
semantic_type: type/FK
dimensions: []
position: 18
-custom_position: 0
database_position: 18
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/tenant_name.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/tenant_name.yaml
index 29fb59db1d8..233bc579350 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/tenant_name.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/tenant_name.yaml
@@ -10,7 +10,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 19
-custom_position: 0
database_position: 19
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/title.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/title.yaml
index 733076bcf6e..2a3bda3d4e8 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/title.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/title.yaml
@@ -13,7 +13,6 @@ effective_type: type/Text
semantic_type: type/Title
dimensions: []
position: 3
-custom_position: 0
database_position: 3
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/total_tokens.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/total_tokens.yaml
index ba03e20c3b2..03fb00698fc 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/total_tokens.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/total_tokens.yaml
@@ -13,7 +13,6 @@ effective_type: type/BigInteger
semantic_type: type/Quantity
dimensions: []
position: 8
-custom_position: 0
database_position: 8
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/user_agent.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/user_agent.yaml
index 074ea5615e3..fd226a5a3fa 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/user_agent.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/user_agent.yaml
@@ -9,7 +9,6 @@ base_type: type/Text
effective_type: type/Text
dimensions: []
position: 23
-custom_position: 0
database_position: 23
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/user_display_name.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/user_display_name.yaml
index 59709aa9f52..c49a7a53ddd 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/user_display_name.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/user_display_name.yaml
@@ -13,7 +13,6 @@ effective_type: type/Text
semantic_type: type/Name
dimensions: []
position: 4
-custom_position: 0
database_position: 4
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/user_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/user_id.yaml
index 7f5ce0ec70b..9e5bbe6d2c4 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/user_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/user_id.yaml
@@ -18,7 +18,6 @@ fk_target_field_id:
- user_id
dimensions: []
position: 2
-custom_position: 0
database_position: 2
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/user_message_count.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/user_message_count.yaml
index 5f5ea6e8de4..aab137951d0 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/user_message_count.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_conversations/fields/user_message_count.yaml
@@ -13,7 +13,6 @@ effective_type: type/BigInteger
semantic_type: type/Quantity
dimensions: []
position: 6
-custom_position: 0
database_position: 6
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_messages/fields/channel_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_messages/fields/channel_id.yaml
index 8dd6ac67f04..718532088ff 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_messages/fields/channel_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_messages/fields/channel_id.yaml
@@ -13,7 +13,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 9
-custom_position: 0
database_position: 9
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_messages/fields/conversation_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_messages/fields/conversation_id.yaml
index 6ac2ef7aaf6..748b47ccb95 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_messages/fields/conversation_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_messages/fields/conversation_id.yaml
@@ -12,7 +12,6 @@ base_type: type/Text
effective_type: type/Text
dimensions: []
position: 1
-custom_position: 0
database_position: 1
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_messages/fields/created_at.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_messages/fields/created_at.yaml
index ac58bfb7928..10ed769cecb 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_messages/fields/created_at.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_messages/fields/created_at.yaml
@@ -13,7 +13,6 @@ effective_type: type/DateTimeWithLocalTZ
semantic_type: type/CreationTimestamp
dimensions: []
position: 2
-custom_position: 0
database_position: 2
serdes/meta:
- id: Internal Metabase Database
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_messages/fields/message_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_messages/fields/message_id.yaml
index 029101bcd70..5df16ad7ad7 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_messages/fields/message_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_messages/fields/message_id.yaml
@@ -13,7 +13,6 @@ effective_type: type/Integer
semantic_type: type/PK
dimensions: []
position: 0
-custom_position: 0
database_position: 0
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_messages/fields/profile_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_messages/fields/profile_id.yaml
index d13bd5ed9b5..57661418789 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_messages/fields/profile_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_messages/fields/profile_id.yaml
@@ -13,7 +13,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 4
-custom_position: 0
database_position: 4
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_messages/fields/role.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_messages/fields/role.yaml
index f76e359172e..69d5195d0ae 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_messages/fields/role.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_messages/fields/role.yaml
@@ -13,7 +13,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 3
-custom_position: 0
database_position: 3
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_messages/fields/slack_msg_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_messages/fields/slack_msg_id.yaml
index 74163c20317..6eb52baf96b 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_messages/fields/slack_msg_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_messages/fields/slack_msg_id.yaml
@@ -12,7 +12,6 @@ base_type: type/Text
effective_type: type/Text
dimensions: []
position: 8
-custom_position: 0
database_position: 8
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_messages/fields/total_tokens.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_messages/fields/total_tokens.yaml
index 1576ac5e3a9..b3d8e7a58f2 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_messages/fields/total_tokens.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_messages/fields/total_tokens.yaml
@@ -13,7 +13,6 @@ effective_type: type/Integer
semantic_type: type/Quantity
dimensions: []
position: 5
-custom_position: 0
database_position: 5
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_messages/fields/user_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_messages/fields/user_id.yaml
index 21828569adf..4cf5d14c8a0 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_messages/fields/user_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_metabot_messages/fields/user_id.yaml
@@ -18,7 +18,6 @@ fk_target_field_id:
- user_id
dimensions: []
position: 6
-custom_position: 0
database_position: 6
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/action_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/action_id.yaml
index 4570241bbdb..fdb30c98324 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/action_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/action_id.yaml
@@ -8,7 +8,6 @@ base_type: type/Integer
effective_type: type/Integer
dimensions: []
position: 16
-custom_position: 0
database_position: 16
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/action_qualified_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/action_qualified_id.yaml
index bd083d7ef7d..01279a80bfb 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/action_qualified_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/action_qualified_id.yaml
@@ -17,7 +17,6 @@ dimensions:
serdes/meta:
- {id: 87cdtRWMXA_ONszqY4pzS, label: action_qualified_id, model: Dimension}
position: 17
-custom_position: 0
database_position: 17
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/auth_method.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/auth_method.yaml
index 6f050e6f281..b0aea0ac560 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/auth_method.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/auth_method.yaml
@@ -8,7 +8,6 @@ base_type: type/Text
effective_type: type/Text
semantic_type: type/Category
position: 28
-custom_position: 0
database_position: 28
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/cache_hit.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/cache_hit.yaml
index 775e6162155..264d8db6623 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/cache_hit.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/cache_hit.yaml
@@ -9,7 +9,6 @@ effective_type: type/Boolean
semantic_type: type/Category
dimensions: []
position: 15
-custom_position: 0
database_position: 15
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/card_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/card_id.yaml
index 510cf877121..94601a994bf 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/card_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/card_id.yaml
@@ -8,7 +8,6 @@ base_type: type/Integer
effective_type: type/Integer
dimensions: []
position: 8
-custom_position: 0
database_position: 8
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/card_qualified_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/card_qualified_id.yaml
index 31302124ab7..2637abf84db 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/card_qualified_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/card_qualified_id.yaml
@@ -17,7 +17,6 @@ dimensions:
serdes/meta:
- {id: 5kq3mRRgC9Ea4OZspf0Hs, label: card_qualified_id, model: Dimension}
position: 9
-custom_position: 0
database_position: 9
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/dashboard_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/dashboard_id.yaml
index 52c9ec1e283..4dba030f4dd 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/dashboard_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/dashboard_id.yaml
@@ -8,7 +8,6 @@ base_type: type/Integer
effective_type: type/Integer
dimensions: []
position: 10
-custom_position: 0
database_position: 10
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/dashboard_qualified_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/dashboard_qualified_id.yaml
index 79e68646fbe..0f325635086 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/dashboard_qualified_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/dashboard_qualified_id.yaml
@@ -17,7 +17,6 @@ dimensions:
serdes/meta:
- {id: LFF3bGrmUaqEW9Q650Bfa, label: dashboard_qualified_id, model: Dimension}
position: 11
-custom_position: 0
database_position: 11
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/database_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/database_id.yaml
index aaff52ba2a8..92ac844064b 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/database_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/database_id.yaml
@@ -8,7 +8,6 @@ base_type: type/Integer
effective_type: type/Integer
dimensions: []
position: 13
-custom_position: 0
database_position: 13
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/database_qualified_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/database_qualified_id.yaml
index a25705924d9..83a26b0e8d9 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/database_qualified_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/database_qualified_id.yaml
@@ -17,7 +17,6 @@ dimensions:
serdes/meta:
- {id: tQ9UVM6UxS0do5krJqfjc, label: database_qualified_id, model: Dimension}
position: 14
-custom_position: 0
database_position: 14
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/embedding_client.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/embedding_client.yaml
index 0828b645bd5..74eb37ade03 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/embedding_client.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/embedding_client.yaml
@@ -15,7 +15,6 @@ dimensions:
serdes/meta:
- {id: nssVx8_sGrHpkHTjIxxK0, label: embedding_client, model: Dimension}
position: 23
-custom_position: 0
database_position: 23
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/embedding_client_identifier.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/embedding_client_identifier.yaml
index 62b4a43a52c..889295a4c79 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/embedding_client_identifier.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/embedding_client_identifier.yaml
@@ -8,7 +8,6 @@ base_type: type/Text
effective_type: type/Text
semantic_type: type/Category
position: 39
-custom_position: 0
database_position: 39
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/embedding_hostname.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/embedding_hostname.yaml
index 96b285e441f..f0afcacc613 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/embedding_hostname.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/embedding_hostname.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 34
-custom_position: 0
database_position: 34
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/embedding_path.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/embedding_path.yaml
index bcbcfd5ae93..95c83440485 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/embedding_path.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/embedding_path.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 35
-custom_position: 0
database_position: 35
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/embedding_route.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/embedding_route.yaml
index 6d63a46e754..36e56d7a433 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/embedding_route.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/embedding_route.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 24
-custom_position: 0
database_position: 24
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/embedding_sdk_version.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/embedding_sdk_version.yaml
index e0d22584d71..3a72aad7615 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/embedding_sdk_version.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/embedding_sdk_version.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 26
-custom_position: 0
database_position: 26
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/entity_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/entity_id.yaml
index 75e8f109cc1..c56d8f8f517 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/entity_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/entity_id.yaml
@@ -9,7 +9,6 @@ effective_type: type/Integer
semantic_type: type/PK
dimensions: []
position: 0
-custom_position: 0
database_position: 0
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/error.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/error.yaml
index 3f51cdc6630..ca5b4aa8493 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/error.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/error.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 6
-custom_position: 0
database_position: 6
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/ip_address.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/ip_address.yaml
index 1163aaf9ffc..999ecf4ea2d 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/ip_address.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/ip_address.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 38
-custom_position: 0
database_position: 38
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/is_db_routed.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/is_db_routed.yaml
index 1f575fa2c1b..b78e317d81d 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/is_db_routed.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/is_db_routed.yaml
@@ -8,7 +8,6 @@ base_type: type/Boolean
effective_type: type/Boolean
dimensions: []
position: 31
-custom_position: 0
database_position: 31
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/is_impersonated.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/is_impersonated.yaml
index c59bcdd7cab..4e0df0485e2 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/is_impersonated.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/is_impersonated.yaml
@@ -8,7 +8,6 @@ base_type: type/Boolean
effective_type: type/Boolean
dimensions: []
position: 30
-custom_position: 0
database_position: 30
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/is_native.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/is_native.yaml
index 071e84a04ff..f7a8cf3ab5d 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/is_native.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/is_native.yaml
@@ -9,7 +9,6 @@ effective_type: type/Boolean
semantic_type: type/Category
dimensions: []
position: 4
-custom_position: 0
database_position: 4
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/is_preview.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/is_preview.yaml
index ea0bdc78581..72e7200b08d 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/is_preview.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/is_preview.yaml
@@ -8,7 +8,6 @@ base_type: type/Boolean
effective_type: type/Boolean
dimensions: []
position: 25
-custom_position: 0
database_position: 25
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/is_sandboxed.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/is_sandboxed.yaml
index cba8e42efe2..a700ed3a3b0 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/is_sandboxed.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/is_sandboxed.yaml
@@ -8,7 +8,6 @@ base_type: type/Boolean
effective_type: type/Boolean
dimensions: []
position: 29
-custom_position: 0
database_position: 29
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/lens_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/lens_id.yaml
index 4838ea15514..5a6dc60f464 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/lens_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/lens_id.yaml
@@ -8,7 +8,6 @@ base_type: type/Text
effective_type: type/Text
dimensions: []
position: 20
-custom_position: 0
database_position: 20
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/lens_params.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/lens_params.yaml
index f794e0e948c..5f2c42b3bb4 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/lens_params.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/lens_params.yaml
@@ -8,7 +8,6 @@ base_type: type/Text
effective_type: type/Text
dimensions: []
position: 21
-custom_position: 0
database_position: 21
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/metabase_version.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/metabase_version.yaml
index dd6c1c8ed75..af102d1cfec 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/metabase_version.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/metabase_version.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 27
-custom_position: 0
database_position: 27
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/parameters.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/parameters.yaml
index 232dc711c21..a43ed559e87 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/parameters.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/parameters.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 32
-custom_position: 0
database_position: 32
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/pulse_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/pulse_id.yaml
index a6ef2038238..ec8c145ea73 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/pulse_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/pulse_id.yaml
@@ -8,7 +8,6 @@ base_type: type/Integer
effective_type: type/Integer
dimensions: []
position: 12
-custom_position: 0
database_position: 12
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/query.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/query.yaml
index c1b0dd20b75..7b982700f46 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/query.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/query.yaml
@@ -10,7 +10,6 @@ semantic_type: type/SerializedJSON
dimensions: []
preview_display: false
position: 22
-custom_position: 0
database_position: 22
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/query_source.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/query_source.yaml
index 00019ee5a7e..91e82417bf6 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/query_source.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/query_source.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Source
dimensions: []
position: 5
-custom_position: 0
database_position: 5
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/result_rows.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/result_rows.yaml
index 1f1c9502f74..f5029621069 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/result_rows.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/result_rows.yaml
@@ -8,7 +8,6 @@ base_type: type/Integer
effective_type: type/Integer
dimensions: []
position: 3
-custom_position: 0
database_position: 3
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/running_time_seconds.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/running_time_seconds.yaml
index 4fb71eec5d6..64b95867385 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/running_time_seconds.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/running_time_seconds.yaml
@@ -8,7 +8,6 @@ base_type: type/Float
effective_type: type/Float
dimensions: []
position: 2
-custom_position: 0
database_position: 2
serdes/meta:
- {id: Internal Metabase Database, model: Database}
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/sanitized_user_agent.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/sanitized_user_agent.yaml
index 4e23d7f6a1a..a064299451a 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/sanitized_user_agent.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/sanitized_user_agent.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 37
-custom_position: 0
database_position: 37
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/started_at.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/started_at.yaml
index e462afecc46..e4e94d499dd 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/started_at.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/started_at.yaml
@@ -9,7 +9,6 @@ effective_type: type/DateTimeWithLocalTZ
semantic_type: type/CreationTimestamp
dimensions: []
position: 1
-custom_position: 0
database_position: 1
serdes/meta:
- {id: Internal Metabase Database, model: Database}
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/tenant_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/tenant_id.yaml
index fafa8fbb083..0b2c435ab04 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/tenant_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/tenant_id.yaml
@@ -17,7 +17,6 @@ dimensions:
serdes/meta:
- {id: SrWQEQU3lWK9qZX0VI2lN, label: tenant, model: Dimension}
position: 33
-custom_position: 0
database_position: 33
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/transform_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/transform_id.yaml
index 9c2119c5657..6929818ca58 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/transform_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/transform_id.yaml
@@ -8,7 +8,6 @@ base_type: type/Integer
effective_type: type/Integer
dimensions: []
position: 18
-custom_position: 0
database_position: 18
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/transform_qualified_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/transform_qualified_id.yaml
index 55d6ce7d44f..624d40fd89b 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/transform_qualified_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/transform_qualified_id.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 19
-custom_position: 0
database_position: 19
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/user_agent.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/user_agent.yaml
index 9d19ce87558..81210c4b323 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/user_agent.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/user_agent.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 36
-custom_position: 0
database_position: 36
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/user_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/user_id.yaml
index 013b83ced06..5be17c6972d 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/user_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/user_id.yaml
@@ -17,7 +17,6 @@ dimensions:
serdes/meta:
- {id: -y4hizlxaFB-MfFB2MOy1, label: user_id, model: Dimension}
position: 7
-custom_position: 0
database_position: 7
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/archived.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/archived.yaml
index 8fe66308847..0f5f3a3822c 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/archived.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/archived.yaml
@@ -9,7 +9,6 @@ effective_type: type/Boolean
semantic_type: type/Category
dimensions: []
position: 5
-custom_position: 0
database_position: 5
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/created_at.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/created_at.yaml
index 9bec1b4700b..e362b35e71c 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/created_at.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/created_at.yaml
@@ -9,7 +9,6 @@ effective_type: type/DateTimeWithLocalTZ
semantic_type: type/CreationTimestamp
dimensions: []
position: 2
-custom_position: 0
database_position: 2
serdes/meta:
- {id: Internal Metabase Database, model: Database}
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/creator_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/creator_id.yaml
index f8162974561..21e6be3a079 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/creator_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/creator_id.yaml
@@ -17,7 +17,6 @@ dimensions:
serdes/meta:
- {id: cEGfE4szUxj3pT8zxBNHC, label: creator_id, model: Dimension}
position: 4
-custom_position: 0
database_position: 4
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/dashboard_qualified_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/dashboard_qualified_id.yaml
index a1d31a54605..4bcbb959cb5 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/dashboard_qualified_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/dashboard_qualified_id.yaml
@@ -17,7 +17,6 @@ dimensions:
serdes/meta:
- {id: tD6rWZbw4DH0Lv6hvqhkQ, label: dashboard_qualified_id, model: Dimension}
position: 6
-custom_position: 0
database_position: 6
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/entity_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/entity_id.yaml
index 267cb6ad632..bfb2aa52399 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/entity_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/entity_id.yaml
@@ -9,7 +9,6 @@ effective_type: type/Integer
semantic_type: type/PK
dimensions: []
position: 0
-custom_position: 0
database_position: 0
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/entity_qualified_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/entity_qualified_id.yaml
index ba53fb837d9..d92c1d8c9f5 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/entity_qualified_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/entity_qualified_id.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/PK
dimensions: []
position: 1
-custom_position: 0
database_position: 1
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/parameters.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/parameters.yaml
index b8f875e0347..e95ea026309 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/parameters.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/parameters.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/SerializedJSON
dimensions: []
position: 13
-custom_position: 0
database_position: 13
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/recipient_external.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/recipient_external.yaml
index 6796626360f..3088afd153a 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/recipient_external.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/recipient_external.yaml
@@ -8,7 +8,6 @@ base_type: type/Text
effective_type: type/Text
dimensions: []
position: 12
-custom_position: 0
database_position: 12
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/recipient_type.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/recipient_type.yaml
index c7fb928bdc0..0b209868405 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/recipient_type.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/recipient_type.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 10
-custom_position: 0
database_position: 10
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/recipients.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/recipients.yaml
index 836a5bead84..3f27c2b8e8d 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/recipients.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/recipients.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 11
-custom_position: 0
database_position: 11
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/schedule_day.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/schedule_day.yaml
index 1b176f897af..06321b51489 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/schedule_day.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/schedule_day.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 8
-custom_position: 0
database_position: 8
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/schedule_hour.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/schedule_hour.yaml
index 8f228544b29..4e5af65cd42 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/schedule_hour.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/schedule_hour.yaml
@@ -9,7 +9,6 @@ effective_type: type/Integer
semantic_type: type/Category
dimensions: []
position: 9
-custom_position: 0
database_position: 9
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/schedule_type.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/schedule_type.yaml
index 461e4255aef..0f7b28d3b31 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/schedule_type.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/schedule_type.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 7
-custom_position: 0
database_position: 7
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/updated_at.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/updated_at.yaml
index 37817512033..103a1552c6a 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/updated_at.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_subscriptions/fields/updated_at.yaml
@@ -9,7 +9,6 @@ effective_type: type/DateTimeWithLocalTZ
semantic_type: type/UpdatedTimestamp
dimensions: []
position: 3
-custom_position: 0
database_position: 3
serdes/meta:
- {id: Internal Metabase Database, model: Database}
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/active.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/active.yaml
index f0aa4dbee70..9d75e6a5b63 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/active.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/active.yaml
@@ -9,7 +9,6 @@ effective_type: type/Boolean
semantic_type: type/Category
dimensions: []
position: 7
-custom_position: 0
database_position: 7
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/created_at.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/created_at.yaml
index dc6fdc2a98e..212891e6b6a 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/created_at.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/created_at.yaml
@@ -9,7 +9,6 @@ effective_type: type/DateTimeWithLocalTZ
semantic_type: type/CreationTimestamp
dimensions: []
position: 2
-custom_position: 0
database_position: 2
serdes/meta:
- {id: Internal Metabase Database, model: Database}
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/database_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/database_id.yaml
index 2f9dd58c9b7..b169322ed08 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/database_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/database_id.yaml
@@ -17,7 +17,6 @@ dimensions:
serdes/meta:
- {id: Wbbn2oPTwM7hn1HjxhCzr, label: database_id, model: Dimension}
position: 8
-custom_position: 0
database_position: 8
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/description.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/description.yaml
index 33e8e37692a..f6fa101bd12 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/description.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/description.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Description
dimensions: []
position: 6
-custom_position: 0
database_position: 6
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/display_name.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/display_name.yaml
index 87cb327296d..fa2c192cbc4 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/display_name.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/display_name.yaml
@@ -8,7 +8,6 @@ base_type: type/Text
effective_type: type/Text
dimensions: []
position: 5
-custom_position: 0
database_position: 5
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/entity_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/entity_id.yaml
index ca15c912d64..9b18d7db399 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/entity_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/entity_id.yaml
@@ -9,7 +9,6 @@ effective_type: type/Integer
semantic_type: type/PK
dimensions: []
position: 0
-custom_position: 0
database_position: 0
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/entity_qualified_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/entity_qualified_id.yaml
index 21da938c270..2b866db3d33 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/entity_qualified_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/entity_qualified_id.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/PK
dimensions: []
position: 1
-custom_position: 0
database_position: 1
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/entity_type.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/entity_type.yaml
index e1e86ec427a..24f37368d1c 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/entity_type.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/entity_type.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 11
-custom_position: 0
database_position: 11
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/estimated_row_count.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/estimated_row_count.yaml
index 18a042a51f4..67c9f746c70 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/estimated_row_count.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/estimated_row_count.yaml
@@ -9,7 +9,6 @@ effective_type: type/BigInteger
semantic_type: type/Quantity
dimensions: []
position: 13
-custom_position: 0
database_position: 13
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/is_upload.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/is_upload.yaml
index d68753650d2..1a44ce38c84 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/is_upload.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/is_upload.yaml
@@ -9,7 +9,6 @@ effective_type: type/Boolean
semantic_type: type/Category
dimensions: []
position: 10
-custom_position: 0
database_position: 10
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/name.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/name.yaml
index df963da0c18..39b947988dd 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/name.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/name.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Name
dimensions: []
position: 4
-custom_position: 0
database_position: 4
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/owner_email.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/owner_email.yaml
index 3a0421889e9..aa3e4b0c9d1 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/owner_email.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/owner_email.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Owner
dimensions: []
position: 15
-custom_position: 0
database_position: 15
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/owner_user_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/owner_user_id.yaml
index 19359e46924..20fb38791d2 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/owner_user_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/owner_user_id.yaml
@@ -9,7 +9,6 @@ effective_type: type/Integer
semantic_type: type/Owner
dimensions: []
position: 16
-custom_position: 0
database_position: 16
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/schema.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/schema.yaml
index 7d326b9f635..bbb1511957e 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/schema.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/schema.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 9
-custom_position: 0
database_position: 9
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/updated_at.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/updated_at.yaml
index 2205f6b873f..266c7c4506a 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/updated_at.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/updated_at.yaml
@@ -9,7 +9,6 @@ effective_type: type/DateTimeWithLocalTZ
semantic_type: type/UpdatedTimestamp
dimensions: []
position: 3
-custom_position: 0
database_position: 3
serdes/meta:
- {id: Internal Metabase Database, model: Database}
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/view_count.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/view_count.yaml
index d47c171ea7f..fd5a06c4090 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/view_count.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/view_count.yaml
@@ -9,7 +9,6 @@ effective_type: type/Integer
semantic_type: type/Quantity
dimensions: []
position: 14
-custom_position: 0
database_position: 14
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/visibility_type.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/visibility_type.yaml
index ddccd1b74d4..c589a3201fe 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/visibility_type.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tables/fields/visibility_type.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 12
-custom_position: 0
database_position: 12
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_task_runs/fields/duration_seconds.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_task_runs/fields/duration_seconds.yaml
index 88ac2299c3b..fc43ebb7b2d 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_task_runs/fields/duration_seconds.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_task_runs/fields/duration_seconds.yaml
@@ -9,7 +9,6 @@ effective_type: type/Float
semantic_type: type/Duration
dimensions: []
position: 7
-custom_position: 0
database_position: 7
serdes/meta:
- {id: Internal Metabase Database, model: Database}
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_task_runs/fields/ended_at.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_task_runs/fields/ended_at.yaml
index 4ecdc037937..53ffd8ae9f7 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_task_runs/fields/ended_at.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_task_runs/fields/ended_at.yaml
@@ -8,7 +8,6 @@ base_type: type/DateTimeWithLocalTZ
effective_type: type/DateTimeWithLocalTZ
dimensions: []
position: 6
-custom_position: 0
database_position: 6
serdes/meta:
- {id: Internal Metabase Database, model: Database}
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_task_runs/fields/entity_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_task_runs/fields/entity_id.yaml
index 391b77feb1c..3164d58c812 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_task_runs/fields/entity_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_task_runs/fields/entity_id.yaml
@@ -8,7 +8,6 @@ base_type: type/Integer
effective_type: type/Integer
dimensions: []
position: 3
-custom_position: 0
database_position: 3
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_task_runs/fields/entity_qualified_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_task_runs/fields/entity_qualified_id.yaml
index d9b6e440014..cdd8d2d1c6f 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_task_runs/fields/entity_qualified_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_task_runs/fields/entity_qualified_id.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 4
-custom_position: 0
database_position: 4
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_task_runs/fields/entity_type.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_task_runs/fields/entity_type.yaml
index 0788487fe39..b51d24df9de 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_task_runs/fields/entity_type.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_task_runs/fields/entity_type.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 2
-custom_position: 0
database_position: 2
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_task_runs/fields/id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_task_runs/fields/id.yaml
index 4542bf3cb5f..34fbbc419f9 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_task_runs/fields/id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_task_runs/fields/id.yaml
@@ -9,7 +9,6 @@ effective_type: type/Integer
semantic_type: type/PK
dimensions: []
position: 0
-custom_position: 0
database_position: 0
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_task_runs/fields/process_uuid.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_task_runs/fields/process_uuid.yaml
index b7f85010452..0748c5e5191 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_task_runs/fields/process_uuid.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_task_runs/fields/process_uuid.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 9
-custom_position: 0
database_position: 9
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_task_runs/fields/run_type.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_task_runs/fields/run_type.yaml
index f9117f1f499..3f95ffbd96e 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_task_runs/fields/run_type.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_task_runs/fields/run_type.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 1
-custom_position: 0
database_position: 1
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_task_runs/fields/started_at.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_task_runs/fields/started_at.yaml
index 5883e98987a..f284772a98b 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_task_runs/fields/started_at.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_task_runs/fields/started_at.yaml
@@ -9,7 +9,6 @@ effective_type: type/DateTimeWithLocalTZ
semantic_type: type/CreationTimestamp
dimensions: []
position: 5
-custom_position: 0
database_position: 5
serdes/meta:
- {id: Internal Metabase Database, model: Database}
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_task_runs/fields/status.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_task_runs/fields/status.yaml
index fe4ed293e61..9677451f629 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_task_runs/fields/status.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_task_runs/fields/status.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 8
-custom_position: 0
database_position: 8
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_task_runs/fields/updated_at.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_task_runs/fields/updated_at.yaml
index 2289aa2ed9d..edabd285c93 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_task_runs/fields/updated_at.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_task_runs/fields/updated_at.yaml
@@ -9,7 +9,6 @@ effective_type: type/DateTimeWithLocalTZ
semantic_type: type/UpdatedTimestamp
dimensions: []
position: 10
-custom_position: 0
database_position: 10
serdes/meta:
- {id: Internal Metabase Database, model: Database}
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tasks/fields/database_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tasks/fields/database_id.yaml
index 68038ea9671..e9cd3fd94a6 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tasks/fields/database_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tasks/fields/database_id.yaml
@@ -10,7 +10,6 @@ effective_type: type/Integer
semantic_type: type/Description
dimensions: []
position: 2
-custom_position: 0
database_position: 2
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tasks/fields/database_qualified_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tasks/fields/database_qualified_id.yaml
index 333096765fd..97127743edb 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tasks/fields/database_qualified_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tasks/fields/database_qualified_id.yaml
@@ -17,7 +17,6 @@ dimensions:
serdes/meta:
- {id: GQmEDdTJkQ_WjQDduKN7G, label: database_qualified_id, model: Dimension}
position: 3
-custom_position: 0
database_position: 3
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tasks/fields/details.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tasks/fields/details.yaml
index 94b55ec5eec..7309d3695bf 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tasks/fields/details.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tasks/fields/details.yaml
@@ -8,7 +8,6 @@ base_type: type/Text
effective_type: type/Text
dimensions: []
position: 7
-custom_position: 0
database_position: 7
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tasks/fields/duration_seconds.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tasks/fields/duration_seconds.yaml
index cfdcb15b9b7..90e5be76563 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tasks/fields/duration_seconds.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tasks/fields/duration_seconds.yaml
@@ -9,7 +9,6 @@ effective_type: type/Float
semantic_type: type/Duration
dimensions: []
position: 6
-custom_position: 0
database_position: 6
serdes/meta:
- {id: Internal Metabase Database, model: Database}
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tasks/fields/ended_at.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tasks/fields/ended_at.yaml
index 0ef17b79c49..384a704d234 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tasks/fields/ended_at.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tasks/fields/ended_at.yaml
@@ -8,7 +8,6 @@ base_type: type/DateTimeWithLocalTZ
effective_type: type/DateTimeWithLocalTZ
dimensions: []
position: 5
-custom_position: 0
database_position: 5
serdes/meta:
- {id: Internal Metabase Database, model: Database}
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tasks/fields/id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tasks/fields/id.yaml
index fdab38f4a2a..d59ccfc44d7 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tasks/fields/id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tasks/fields/id.yaml
@@ -9,7 +9,6 @@ effective_type: type/Integer
semantic_type: type/PK
dimensions: []
position: 0
-custom_position: 0
database_position: 0
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tasks/fields/run_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tasks/fields/run_id.yaml
index c6a405929be..cc6c50bed2c 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tasks/fields/run_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tasks/fields/run_id.yaml
@@ -10,7 +10,6 @@ semantic_type: type/FK
fk_target_field_id: [Internal Metabase Database, public, v_task_runs, id]
dimensions: []
position: 8
-custom_position: 0
database_position: 8
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tasks/fields/started_at.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tasks/fields/started_at.yaml
index c54e2859ac8..962ea1ea1dd 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tasks/fields/started_at.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tasks/fields/started_at.yaml
@@ -9,7 +9,6 @@ effective_type: type/DateTimeWithLocalTZ
semantic_type: type/CreationTimestamp
dimensions: []
position: 4
-custom_position: 0
database_position: 4
serdes/meta:
- {id: Internal Metabase Database, model: Database}
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tasks/fields/status.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tasks/fields/status.yaml
index 2ace6f6bed6..90ec988aee1 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tasks/fields/status.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tasks/fields/status.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 2
-custom_position: 0
database_position: 2
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tasks/fields/task.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tasks/fields/task.yaml
index adc20cff044..550926b3d1f 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tasks/fields/task.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tasks/fields/task.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 1
-custom_position: 0
database_position: 1
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tenants/fields/attributes.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tenants/fields/attributes.yaml
index 9888ce199d8..13b2ff44737 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tenants/fields/attributes.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tenants/fields/attributes.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 5
-custom_position: 0
database_position: 5
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tenants/fields/created_at.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tenants/fields/created_at.yaml
index bc540ff11cb..5638897c6f8 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tenants/fields/created_at.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tenants/fields/created_at.yaml
@@ -9,7 +9,6 @@ effective_type: type/DateTimeWithLocalTZ
semantic_type: type/CreationTimestamp
dimensions: []
position: 6
-custom_position: 0
database_position: 6
serdes/meta:
- {id: Internal Metabase Database, model: Database}
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tenants/fields/entity_qualified_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tenants/fields/entity_qualified_id.yaml
index c2cd8908c26..21f0339a2ce 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tenants/fields/entity_qualified_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tenants/fields/entity_qualified_id.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 1
-custom_position: 0
database_position: 1
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tenants/fields/is_active.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tenants/fields/is_active.yaml
index 816a3e15535..dd715fd3e4c 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tenants/fields/is_active.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tenants/fields/is_active.yaml
@@ -8,7 +8,6 @@ base_type: type/Boolean
effective_type: type/Boolean
dimensions: []
position: 4
-custom_position: 0
database_position: 4
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tenants/fields/name.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tenants/fields/name.yaml
index 0922fb622bc..a1b8889bc66 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tenants/fields/name.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tenants/fields/name.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Name
dimensions: []
position: 2
-custom_position: 0
database_position: 2
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tenants/fields/slug.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tenants/fields/slug.yaml
index ded06bd5b74..9de4c79aaf1 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tenants/fields/slug.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tenants/fields/slug.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 3
-custom_position: 0
database_position: 3
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tenants/fields/tenant_collection_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tenants/fields/tenant_collection_id.yaml
index 9a15f2fab56..c0ea51bb056 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tenants/fields/tenant_collection_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tenants/fields/tenant_collection_id.yaml
@@ -8,7 +8,6 @@ base_type: type/Integer
effective_type: type/Integer
dimensions: []
position: 8
-custom_position: 0
database_position: 8
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tenants/fields/tenant_collection_qualified_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tenants/fields/tenant_collection_qualified_id.yaml
index 5a5b677fa53..b68bcc09d12 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tenants/fields/tenant_collection_qualified_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tenants/fields/tenant_collection_qualified_id.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 9
-custom_position: 0
database_position: 9
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tenants/fields/tenant_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tenants/fields/tenant_id.yaml
index fb527e870e0..9715298f7d8 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tenants/fields/tenant_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tenants/fields/tenant_id.yaml
@@ -8,7 +8,6 @@ base_type: type/Integer
effective_type: type/Integer
dimensions: []
position: 0
-custom_position: 0
database_position: 0
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tenants/fields/updated_at.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tenants/fields/updated_at.yaml
index 1291a4814e7..7cb8206527d 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tenants/fields/updated_at.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_tenants/fields/updated_at.yaml
@@ -9,7 +9,6 @@ effective_type: type/DateTimeWithLocalTZ
semantic_type: type/UpdatedTimestamp
dimensions: []
position: 7
-custom_position: 0
database_position: 7
serdes/meta:
- {id: Internal Metabase Database, model: Database}
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/date_joined.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/date_joined.yaml
index d20482e285c..325ef563c4a 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/date_joined.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/date_joined.yaml
@@ -9,7 +9,6 @@ effective_type: type/DateTimeWithLocalTZ
semantic_type: type/JoinTimestamp
dimensions: []
position: 7
-custom_position: 0
database_position: 7
serdes/meta:
- {id: Internal Metabase Database, model: Database}
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/email.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/email.yaml
index e6324eb73ab..bc181d3b4fe 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/email.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/email.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Email
dimensions: []
position: 3
-custom_position: 0
database_position: 3
has_field_values: list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/entity_qualified_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/entity_qualified_id.yaml
index 7b893e4e0d0..13ad8995962 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/entity_qualified_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/entity_qualified_id.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/PK
dimensions: []
position: 1
-custom_position: 0
database_position: 1
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/first_name.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/first_name.yaml
index 6794b0859ae..03c7cc5c925 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/first_name.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/first_name.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Name
dimensions: []
position: 4
-custom_position: 0
database_position: 4
has_field_values: list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/full_name.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/full_name.yaml
index 70afdf058b1..1044528f925 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/full_name.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/full_name.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Name
dimensions: []
position: 6
-custom_position: 0
database_position: 6
has_field_values: list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/is_active.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/is_active.yaml
index d066dd687b4..b59cc22200e 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/is_active.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/is_active.yaml
@@ -9,7 +9,6 @@ effective_type: type/Boolean
semantic_type: type/Category
dimensions: []
position: 11
-custom_position: 0
database_position: 11
has_field_values: list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/is_admin.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/is_admin.yaml
index 2ea28a8c856..0ee3e3fc125 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/is_admin.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/is_admin.yaml
@@ -9,7 +9,6 @@ effective_type: type/Boolean
semantic_type: type/Category
dimensions: []
position: 10
-custom_position: 0
database_position: 10
has_field_values: list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/last_login.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/last_login.yaml
index 568aa8b7708..b163ff12f65 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/last_login.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/last_login.yaml
@@ -8,7 +8,6 @@ base_type: type/DateTimeWithLocalTZ
effective_type: type/DateTimeWithLocalTZ
dimensions: []
position: 8
-custom_position: 0
database_position: 8
serdes/meta:
- {id: Internal Metabase Database, model: Database}
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/last_name.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/last_name.yaml
index b4bf7c6f0db..ec862c49e4c 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/last_name.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/last_name.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Name
dimensions: []
position: 5
-custom_position: 0
database_position: 5
has_field_values: list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/locale.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/locale.yaml
index 5f6afafb7d3..95b8f251d50 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/locale.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/locale.yaml
@@ -8,7 +8,6 @@ base_type: type/Text
effective_type: type/Text
dimensions: []
position: 13
-custom_position: 0
database_position: 13
has_field_values: list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/sso_source.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/sso_source.yaml
index 53672b4eb97..fe0a1b63e05 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/sso_source.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/sso_source.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Source
dimensions: []
position: 12
-custom_position: 0
database_position: 12
has_field_values: list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/tenant_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/tenant_id.yaml
index e10550a7a77..b2fe39d3fb1 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/tenant_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/tenant_id.yaml
@@ -17,7 +17,6 @@ dimensions:
serdes/meta:
- {id: v5tnim5gwIE3HfB07NEqG, label: tenant, model: Dimension}
position: 14
-custom_position: 0
database_position: 14
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/tenant_qualified_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/tenant_qualified_id.yaml
index e40e2f2f5e2..9cbaeff277f 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/tenant_qualified_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/tenant_qualified_id.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 15
-custom_position: 0
database_position: 15
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/type.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/type.yaml
index 28fee60a6a0..28019bc90e4 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/type.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/type.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 2
-custom_position: 0
database_position: 2
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/updated_at.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/updated_at.yaml
index 419cb020fb2..5365184d502 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/updated_at.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/updated_at.yaml
@@ -9,7 +9,6 @@ effective_type: type/DateTimeWithLocalTZ
semantic_type: type/UpdatedTimestamp
dimensions: []
position: 9
-custom_position: 0
database_position: 9
serdes/meta:
- {id: Internal Metabase Database, model: Database}
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/user_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/user_id.yaml
index f80fb195ab1..c8d62f55bec 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/user_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_users/fields/user_id.yaml
@@ -9,7 +9,6 @@ effective_type: type/Integer
semantic_type: type/PK
dimensions: []
position: 0
-custom_position: 0
database_position: 0
has_field_values: none
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/auth_method.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/auth_method.yaml
index 09d0a080168..9ccd2ba11fb 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/auth_method.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/auth_method.yaml
@@ -8,7 +8,6 @@ base_type: type/Text
effective_type: type/Text
semantic_type: type/Category
position: 13
-custom_position: 0
database_position: 13
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/context.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/context.yaml
index 7b07d694b87..4349f91a1fe 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/context.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/context.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 10
-custom_position: 0
database_position: 10
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/details.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/details.yaml
index 8b45e3338b7..3d32966ba1d 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/details.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/details.yaml
@@ -10,7 +10,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 6
-custom_position: 0
database_position: 6
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/embedding_client.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/embedding_client.yaml
index bed326c6e83..afde79fc04a 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/embedding_client.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/embedding_client.yaml
@@ -15,7 +15,6 @@ dimensions:
serdes/meta:
- {id: Q28WWBnVpviA9FlQnjpMo, label: embedding_client, model: Dimension}
position: 6
-custom_position: 0
database_position: 6
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/embedding_client_identifier.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/embedding_client_identifier.yaml
index c6f189494c6..95ca2639837 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/embedding_client_identifier.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/embedding_client_identifier.yaml
@@ -8,7 +8,6 @@ base_type: type/Text
effective_type: type/Text
semantic_type: type/Category
position: 20
-custom_position: 0
database_position: 20
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/embedding_hostname.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/embedding_hostname.yaml
index e8fa98a931b..d46bbee507c 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/embedding_hostname.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/embedding_hostname.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 15
-custom_position: 0
database_position: 15
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/embedding_path.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/embedding_path.yaml
index d40dbd2156a..fc09736e808 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/embedding_path.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/embedding_path.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 16
-custom_position: 0
database_position: 16
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/embedding_route.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/embedding_route.yaml
index acb945482fe..6343a7c1ec3 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/embedding_route.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/embedding_route.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 7
-custom_position: 0
database_position: 7
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/embedding_sdk_version.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/embedding_sdk_version.yaml
index 33dffac1d19..756e7c7f9eb 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/embedding_sdk_version.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/embedding_sdk_version.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 11
-custom_position: 0
database_position: 11
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/entity_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/entity_id.yaml
index 7eb1613337b..0eac2babcdb 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/entity_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/entity_id.yaml
@@ -8,7 +8,6 @@ base_type: type/Integer
effective_type: type/Integer
dimensions: []
position: 4
-custom_position: 0
database_position: 4
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/entity_qualified_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/entity_qualified_id.yaml
index 1873a664cc7..d360f20e3b4 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/entity_qualified_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/entity_qualified_id.yaml
@@ -17,7 +17,6 @@ dimensions:
serdes/meta:
- {id: GLCMnQtynL0QeyQBNvKy9, label: entity_qualified_id, model: Dimension}
position: 5
-custom_position: 0
database_position: 5
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/entity_type.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/entity_type.yaml
index e2949dfc868..93a78881d37 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/entity_type.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/entity_type.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 3
-custom_position: 0
database_position: 3
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/has_access.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/has_access.yaml
index 7964c4dc419..5a5c3feb918 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/has_access.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/has_access.yaml
@@ -8,7 +8,6 @@ base_type: type/Boolean
effective_type: type/Boolean
dimensions: []
position: 9
-custom_position: 0
database_position: 9
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/id.yaml
index 17e9447b681..b9b4d755ee7 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/id.yaml
@@ -9,7 +9,6 @@ effective_type: type/Integer
semantic_type: type/PK
dimensions: []
position: 0
-custom_position: 0
database_position: 0
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/ip_address.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/ip_address.yaml
index 115094634b3..e3b8afdc6dc 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/ip_address.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/ip_address.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 19
-custom_position: 0
database_position: 19
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/is_preview.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/is_preview.yaml
index 5a4e1a211ef..6eeeb2e581c 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/is_preview.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/is_preview.yaml
@@ -8,7 +8,6 @@ base_type: type/Boolean
effective_type: type/Boolean
dimensions: []
position: 8
-custom_position: 0
database_position: 8
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/metabase_version.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/metabase_version.yaml
index 264ffdf2f6f..2ce8dcd6850 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/metabase_version.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/metabase_version.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 12
-custom_position: 0
database_position: 12
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/sanitized_user_agent.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/sanitized_user_agent.yaml
index b88ced13399..0682e66f2b4 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/sanitized_user_agent.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/sanitized_user_agent.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 18
-custom_position: 0
database_position: 18
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/tenant_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/tenant_id.yaml
index 3e6253d88c7..6c0b0feb7af 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/tenant_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/tenant_id.yaml
@@ -17,7 +17,6 @@ dimensions:
serdes/meta:
- {id: gema7Cwa0xbZfs1jbUuNJ, label: tenant, model: Dimension}
position: 14
-custom_position: 0
database_position: 14
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/timestamp.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/timestamp.yaml
index 6760c66d8fd..ed472323548 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/timestamp.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/timestamp.yaml
@@ -9,7 +9,6 @@ effective_type: type/DateTimeWithLocalTZ
semantic_type: type/CreationTimestamp
dimensions: []
position: 1
-custom_position: 0
database_position: 1
serdes/meta:
- {id: Internal Metabase Database, model: Database}
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/user_agent.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/user_agent.yaml
index 186633bb3ae..e74f5c66498 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/user_agent.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/user_agent.yaml
@@ -9,7 +9,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 17
-custom_position: 0
database_position: 17
has_field_values: auto-list
serdes/meta:
diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/user_id.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/user_id.yaml
index bb5070d60f8..0c2c430cd4c 100644
--- a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/user_id.yaml
+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_view_log/fields/user_id.yaml
@@ -17,7 +17,6 @@ dimensions:
serdes/meta:
- {id: 0s2Cvg3Ih5DbwioUzbdQZ, label: user_id, model: Dimension}
position: 2
-custom_position: 0
database_position: 2
has_field_values: auto-list
serdes/meta:
diff --git a/resources/migrations/064/20260909_field_user_settings_set_flags.yaml b/resources/migrations/064/20260909_field_user_settings_set_flags.yaml
new file mode 100644
index 00000000000..7b61837768e
--- /dev/null
+++ b/resources/migrations/064/20260909_field_user_settings_set_flags.yaml
@@ -0,0 +1,162 @@
+## Record that a user set a nullable Field column to NULL on purpose.
+##
+## A NULL in metabase_field_user_settings normally means "not set by a user". description, semantic_type and
+## fk_target_field_id are nullable on metabase_field and also written by sync, so for those a user's NULL must still
+## win over the sync value: these flags say whether the user made the call.
+
+databaseChangeLog:
+ - changeSet:
+ id: v64.2026-09-09T00:00:00
+ author: ranquild
+ comment: Add description_set to metabase_field_user_settings
+ changes:
+ - addColumn:
+ tableName: metabase_field_user_settings
+ columns:
+ - column:
+ name: description_set
+ type: ${boolean.type}
+ defaultValueBoolean: false
+ remarks: "Whether the user set description, so a NULL description overrides the sync value."
+ constraints:
+ nullable: false
+
+ - changeSet:
+ id: v64.2026-09-09T00:00:01
+ author: ranquild
+ comment: Add semantic_type_set to metabase_field_user_settings
+ changes:
+ - addColumn:
+ tableName: metabase_field_user_settings
+ columns:
+ - column:
+ name: semantic_type_set
+ type: ${boolean.type}
+ defaultValueBoolean: false
+ remarks: "Whether the user set semantic_type, so a NULL semantic_type overrides the sync value."
+ constraints:
+ nullable: false
+
+ - changeSet:
+ id: v64.2026-09-09T00:00:02
+ author: ranquild
+ comment: Add fk_target_field_id_set to metabase_field_user_settings
+ changes:
+ - addColumn:
+ tableName: metabase_field_user_settings
+ columns:
+ - column:
+ name: fk_target_field_id_set
+ type: ${boolean.type}
+ defaultValueBoolean: false
+ remarks: "Whether the user set fk_target_field_id, so a NULL fk_target_field_id overrides the sync value."
+ constraints:
+ nullable: false
+
+ - changeSet:
+ id: v64.2026-09-09T00:00:03
+ author: ranquild
+ comment: Mark the description, semantic_type and fk_target_field_id user settings already recorded as set
+ changes:
+ - sql:
+ sql: >-
+ UPDATE metabase_field_user_settings
+ SET description_set = (description IS NOT NULL),
+ semantic_type_set = (semantic_type IS NOT NULL),
+ fk_target_field_id_set = (fk_target_field_id IS NOT NULL)
+ rollback: # not necessary
+
+ - changeSet:
+ id: v64.2026-09-09T00:00:04
+ author: ranquild
+ comment: Update v_fields to show display_name, description, visibility_type, fk_target_field_id and
+ has_field_values as users see them, overlaying metabase_field_user_settings
+ changes:
+ - sqlFile:
+ dbms: postgresql
+ path: ../instance_analytics_views/fields/v3/postgres-fields.sql
+ relativeToChangelogFile: true
+ - sqlFile:
+ dbms: mysql,mariadb
+ path: ../instance_analytics_views/fields/v3/mysql-fields.sql
+ relativeToChangelogFile: true
+ - sqlFile:
+ dbms: h2
+ path: ../instance_analytics_views/fields/v3/h2-fields.sql
+ relativeToChangelogFile: true
+ rollback:
+ - sqlFile:
+ dbms: postgresql
+ path: ../instance_analytics_views/fields/v1/postgres-fields.sql
+ relativeToChangelogFile: true
+ - sqlFile:
+ dbms: mysql,mariadb
+ path: ../instance_analytics_views/fields/v2/mysql-fields.sql
+ relativeToChangelogFile: true
+ - sqlFile:
+ dbms: h2
+ path: ../instance_analytics_views/fields/v1/h2-fields.sql
+ relativeToChangelogFile: true
+
+ - changeSet:
+ id: v64.2026-09-09T00:00:05
+ author: ranquild
+ comment: Clear the metabase_field copies of the values held in metabase_field_user_settings
+ changes:
+ - sql:
+ dbms: postgresql
+ sql: >-
+ UPDATE metabase_field AS f
+ SET caveats = CASE WHEN s.caveats IS NULL THEN f.caveats ELSE NULL END,
+ points_of_interest = CASE WHEN s.points_of_interest IS NULL THEN f.points_of_interest ELSE NULL END,
+ has_field_values = CASE WHEN s.has_field_values IS NULL THEN f.has_field_values ELSE NULL END,
+ settings = CASE WHEN s.settings IS NULL THEN f.settings ELSE NULL END
+ FROM metabase_field_user_settings AS s
+ WHERE s.field_id = f.id
+ - sql:
+ dbms: mysql,mariadb
+ sql: >-
+ UPDATE metabase_field f
+ JOIN metabase_field_user_settings s ON s.field_id = f.id
+ SET f.caveats = CASE WHEN s.caveats IS NULL THEN f.caveats ELSE NULL END,
+ f.points_of_interest = CASE WHEN s.points_of_interest IS NULL THEN f.points_of_interest ELSE NULL END,
+ f.has_field_values = CASE WHEN s.has_field_values IS NULL THEN f.has_field_values ELSE NULL END,
+ f.settings = CASE WHEN s.settings IS NULL THEN f.settings ELSE NULL END
+ - sql:
+ dbms: h2
+ sql: >-
+ UPDATE metabase_field
+ SET caveats = CASE WHEN EXISTS (SELECT 1 FROM metabase_field_user_settings s WHERE s.field_id = metabase_field.id AND s.caveats IS NOT NULL) THEN NULL ELSE caveats END,
+ points_of_interest = CASE WHEN EXISTS (SELECT 1 FROM metabase_field_user_settings s WHERE s.field_id = metabase_field.id AND s.points_of_interest IS NOT NULL) THEN NULL ELSE points_of_interest END,
+ has_field_values = CASE WHEN EXISTS (SELECT 1 FROM metabase_field_user_settings s WHERE s.field_id = metabase_field.id AND s.has_field_values IS NOT NULL) THEN NULL ELSE has_field_values END,
+ settings = CASE WHEN EXISTS (SELECT 1 FROM metabase_field_user_settings s WHERE s.field_id = metabase_field.id AND s.settings IS NOT NULL) THEN NULL ELSE settings END
+ WHERE EXISTS (SELECT 1 FROM metabase_field_user_settings s WHERE s.field_id = metabase_field.id)
+ rollback:
+ - sql:
+ dbms: postgresql
+ sql: >-
+ UPDATE metabase_field AS f
+ SET caveats = COALESCE(s.caveats, f.caveats),
+ points_of_interest = COALESCE(s.points_of_interest, f.points_of_interest),
+ has_field_values = COALESCE(s.has_field_values, f.has_field_values),
+ settings = COALESCE(s.settings, f.settings)
+ FROM metabase_field_user_settings AS s
+ WHERE s.field_id = f.id
+ - sql:
+ dbms: mysql,mariadb
+ sql: >-
+ UPDATE metabase_field f
+ JOIN metabase_field_user_settings s ON s.field_id = f.id
+ SET f.caveats = COALESCE(s.caveats, f.caveats),
+ f.points_of_interest = COALESCE(s.points_of_interest, f.points_of_interest),
+ f.has_field_values = COALESCE(s.has_field_values, f.has_field_values),
+ f.settings = COALESCE(s.settings, f.settings)
+ - sql:
+ dbms: h2
+ sql: >-
+ UPDATE metabase_field
+ SET caveats = COALESCE((SELECT s.caveats FROM metabase_field_user_settings s WHERE s.field_id = metabase_field.id), caveats),
+ points_of_interest = COALESCE((SELECT s.points_of_interest FROM metabase_field_user_settings s WHERE s.field_id = metabase_field.id), points_of_interest),
+ has_field_values = COALESCE((SELECT s.has_field_values FROM metabase_field_user_settings s WHERE s.field_id = metabase_field.id), has_field_values),
+ settings = COALESCE((SELECT s.settings FROM metabase_field_user_settings s WHERE s.field_id = metabase_field.id), settings)
+ WHERE EXISTS (SELECT 1 FROM metabase_field_user_settings s WHERE s.field_id = metabase_field.id)
diff --git a/resources/migrations/064/20260911_table_user_settings.yaml b/resources/migrations/064/20260911_table_user_settings.yaml
new file mode 100644
index 00000000000..10820df5f71
--- /dev/null
+++ b/resources/migrations/064/20260911_table_user_settings.yaml
@@ -0,0 +1,473 @@
+## Adds metabase_table_user_settings, the mirror of metabase_table recording the values a user set
+## through the UI, as metabase_field_user_settings does for Fields. metabase_table keeps sync's
+## values; readers merge the two (see metabase.warehouse-schema-overlay.core/table-query).
+##
+## A NULL here normally means "the user did not set this". For a column that is nullable on
+## metabase_table a user's NULL is indistinguishable from that, so those carry a `_set` flag saying
+## the user made the call. collection_id is the exception: it only means anything alongside
+## is_published, which is NOT NULL, so is_published answers for the pair.
+
+databaseChangeLog:
+ - changeSet:
+ id: v64.2026-09-11T00:00:00
+ author: ranquild
+ comment: Add metabase_table_user_settings
+ changes:
+ - createTable:
+ tableName: metabase_table_user_settings
+ remarks: The values a user set on a Table; metabase_table keeps the ones sync wrote
+ columns:
+ - column:
+ remarks: The related Table
+ name: table_id
+ type: int
+ constraints:
+ primaryKey: true
+ nullable: false
+ referencedTableName: metabase_table
+ referencedColumnNames: id
+ foreignKeyName: fk_table_user_settings_table_id
+ deleteCascade: true
+ - column:
+ name: created_at
+ remarks: The timestamp of when the user setting was created
+ type: ${timestamp_type}
+ defaultValueComputed: current_timestamp
+ constraints:
+ nullable: false
+ - column:
+ name: updated_at
+ remarks: The timestamp of when the user setting was updated
+ type: ${timestamp_type}
+ defaultValueComputed: current_timestamp
+ constraints:
+ nullable: false
+ - column:
+ remarks: User-set display_name for the Table
+ name: display_name
+ type: varchar(256)
+ - column:
+ remarks: User-set description for the Table
+ name: description
+ type: ${text.type}
+ - column:
+ remarks: User-set entity_type for the Table
+ name: entity_type
+ type: varchar(254)
+ - column:
+ remarks: User-set visibility_type for the Table
+ name: visibility_type
+ type: varchar(254)
+ - column:
+ remarks: User-set caveats for the Table
+ name: caveats
+ type: ${text.type}
+ - column:
+ remarks: User-set points_of_interest for the Table
+ name: points_of_interest
+ type: ${text.type}
+ - column:
+ remarks: User-set data_layer for the Table
+ name: data_layer
+ type: varchar(254)
+ - column:
+ remarks: User-set data_source for the Table
+ name: data_source
+ type: varchar(254)
+ - column:
+ remarks: User-set owner_email for the Table
+ name: owner_email
+ type: ${text.type}
+ - column:
+ remarks: User-set owner_user_id for the Table
+ name: owner_user_id
+ type: int
+ - column:
+ remarks: User-set field_order for the Table; NOT NULL on metabase_table, so a NULL here means unset
+ name: field_order
+ type: varchar(254)
+ - column:
+ remarks: User-set show_in_getting_started for the Table; NOT NULL on metabase_table, so a NULL here means unset
+ name: show_in_getting_started
+ type: ${boolean.type}
+ - column:
+ remarks: User-set data_authority for the Table; NOT NULL on metabase_table, so a NULL here means unset
+ name: data_authority
+ type: varchar(20)
+ - column:
+ remarks: User-set is_published for the Table; NOT NULL on metabase_table, so a NULL here means unset
+ name: is_published
+ type: ${boolean.type}
+ - column:
+ remarks: User-set collection_id for the Table; set when is_published is, which answers for the pair
+ name: collection_id
+ type: int
+ - column:
+ remarks: Whether the user set description, so a NULL description overrides the sync value
+ name: description_set
+ type: ${boolean.type}
+ defaultValueBoolean: false
+ constraints:
+ nullable: false
+ - column:
+ remarks: Whether the user set visibility_type, so a NULL visibility_type overrides the sync value
+ name: visibility_type_set
+ type: ${boolean.type}
+ defaultValueBoolean: false
+ constraints:
+ nullable: false
+ - column:
+ remarks: Whether the user set caveats, so a NULL caveats overrides the sync value
+ name: caveats_set
+ type: ${boolean.type}
+ defaultValueBoolean: false
+ constraints:
+ nullable: false
+ - column:
+ remarks: Whether the user set points_of_interest, so a NULL points_of_interest overrides the sync value
+ name: points_of_interest_set
+ type: ${boolean.type}
+ defaultValueBoolean: false
+ constraints:
+ nullable: false
+ - column:
+ remarks: Whether the user set data_layer, so a NULL data_layer overrides the sync value
+ name: data_layer_set
+ type: ${boolean.type}
+ defaultValueBoolean: false
+ constraints:
+ nullable: false
+ - column:
+ remarks: Whether the user set data_source, so a NULL data_source overrides the sync value
+ name: data_source_set
+ type: ${boolean.type}
+ defaultValueBoolean: false
+ constraints:
+ nullable: false
+
+ ## The indexes come before the FK constraints: MySQL/MariaDB require an index behind every FK and
+ ## reuse an existing one, so this keeps rollback working (the FK is dropped before the index it
+ ## would otherwise still need). Postgres does not auto-index FKs, which fks-are-indexed-test checks.
+ - changeSet:
+ id: v64.2026-09-11T00:00:01
+ author: ranquild
+ comment: Index metabase_table_user_settings.collection_id
+ changes:
+ - createIndex:
+ tableName: metabase_table_user_settings
+ indexName: idx_table_user_settings_collection_id
+ columns:
+ - column:
+ name: collection_id
+
+ - changeSet:
+ id: v64.2026-09-11T00:00:02
+ author: ranquild
+ comment: Add FK constraint on metabase_table_user_settings.collection_id
+ changes:
+ - addForeignKeyConstraint:
+ baseTableName: metabase_table_user_settings
+ baseColumnNames: collection_id
+ referencedTableName: collection
+ referencedColumnNames: id
+ constraintName: fk_table_user_settings_collection_id
+ onDelete: SET NULL
+
+ - changeSet:
+ id: v64.2026-09-11T00:00:03
+ author: ranquild
+ comment: Index metabase_table_user_settings.owner_user_id
+ changes:
+ - createIndex:
+ tableName: metabase_table_user_settings
+ indexName: idx_table_user_settings_owner_user_id
+ columns:
+ - column:
+ name: owner_user_id
+
+ - changeSet:
+ id: v64.2026-09-11T00:00:04
+ author: ranquild
+ comment: Add FK constraint on metabase_table_user_settings.owner_user_id
+ changes:
+ - addForeignKeyConstraint:
+ baseTableName: metabase_table_user_settings
+ baseColumnNames: owner_user_id
+ referencedTableName: core_user
+ referencedColumnNames: id
+ constraintName: fk_table_user_settings_owner_user_id
+ onDelete: SET NULL
+
+ ## Backfill, best effort: a published Table's metadata is treated as the user's wherever it cannot be
+ ## attributed to sync, so an existing instance keeps showing what it shows today.
+ ##
+ ## Only published Tables: those are the ones git sync carries, and a settings row for every active Table
+ ## would have it write a file per Table -- most of them recording nothing a user ever did. An unpublished
+ ## Table needs no row: with none, every column falls through to metabase_table, which is what it shows
+ ## today, and the first user edit creates the row.
+ ##
+ ## display_name is copied unconditionally: it has no `_set` flag (the user cannot clear it, so a
+ ## non-NULL value is itself the signal -- the humanization strategy is env-configurable anyway, so
+ ## there is no default to compare against). The purely user-authored caveats and points_of_interest,
+ ## and owner_email and owner_user_id, are copied and marked set. description is copied and marked set
+ ## only when it is non-NULL: it may be user-written or synced from a database comment, and a user
+ ## clearing a synced description is far rarer than never having written one, so NULL reads as "not set".
+ ##
+ ## Recorded only when they differ from the value the system writes: data_layer 'internal',
+ ## data_authority 'unconfigured', field_order 'database' (every in-tree driver's
+ ## default-field-order), visibility_type NULL or the sync-authored 'cruft', and
+ ## is_published/show_in_getting_started FALSE. collection_id rides along with is_published.
+ ##
+ ## entity_type and data_source stay unset: the sync classifier writes a name-derived entity_type on
+ ## every Table, and uploads and transforms write data_source, so recording either would pin the
+ ## system's own output as the user's.
+ ##
+ ## Inactive Tables are skipped, and the audit/instance-analytics database is excluded (Metabase
+ ## curates it), via NOT EXISTS rather than NOT IN (SELECT ...) since NOT IN degrades on large
+ ## instances (#78271). Rows that already exist are left alone, and created_at/updated_at are
+ ## omitted so their current_timestamp defaults apply.
+ - changeSet:
+ id: v64.2026-09-11T00:00:05
+ author: ranquild
+ comment: Backfill metabase_table_user_settings for active published tables
+ changes:
+ - sql:
+ sql: >-
+ INSERT INTO metabase_table_user_settings
+ (table_id, display_name, description, description_set,
+ caveats, caveats_set, points_of_interest, points_of_interest_set,
+ owner_email, owner_user_id,
+ visibility_type, visibility_type_set, data_layer, data_layer_set,
+ data_authority, field_order, show_in_getting_started, is_published, collection_id,
+ created_at, updated_at)
+ SELECT
+ t.id,
+ t.display_name,
+ t.description,
+ CASE WHEN t.description IS NOT NULL THEN TRUE ELSE FALSE END,
+ t.caveats,
+ CASE WHEN t.caveats IS NOT NULL THEN TRUE ELSE FALSE END,
+ t.points_of_interest,
+ CASE WHEN t.points_of_interest IS NOT NULL THEN TRUE ELSE FALSE END,
+ t.owner_email,
+ t.owner_user_id,
+ CASE WHEN t.visibility_type <> 'cruft' THEN t.visibility_type END,
+ CASE WHEN t.visibility_type IS NOT NULL AND t.visibility_type <> 'cruft' THEN TRUE ELSE FALSE END,
+ CASE WHEN t.data_layer <> 'internal' THEN t.data_layer END,
+ CASE WHEN t.data_layer IS NOT NULL AND t.data_layer <> 'internal' THEN TRUE ELSE FALSE END,
+ CASE WHEN t.data_authority <> 'unconfigured' THEN t.data_authority END,
+ CASE WHEN t.field_order <> 'database' THEN t.field_order END,
+ CASE WHEN t.show_in_getting_started = TRUE THEN t.show_in_getting_started END,
+ CASE WHEN t.is_published = TRUE THEN t.is_published END,
+ CASE WHEN t.is_published = TRUE THEN t.collection_id END,
+ t.created_at,
+ t.updated_at
+ FROM metabase_table t
+ WHERE t.active = TRUE
+ AND t.is_published = TRUE
+ AND NOT EXISTS (SELECT 1 FROM metabase_database d WHERE d.id = t.db_id AND d.is_audit = TRUE)
+ AND NOT EXISTS (SELECT 1 FROM metabase_table_user_settings s WHERE s.table_id = t.id)
+ rollback: # not necessary; the table is dropped by the rollback of v64.2026-09-11T00:00:00
+
+ - changeSet:
+ id: v64.2026-09-11T00:00:06
+ author: ranquild
+ comment: Clear the user-set columns the backfill moved into metabase_table_user_settings
+ changes:
+ - sql:
+ dbms: postgresql
+ sql: >-
+ UPDATE metabase_table AS t
+ SET caveats = NULL,
+ points_of_interest = NULL,
+ owner_email = NULL,
+ owner_user_id = NULL,
+ is_published = FALSE,
+ collection_id = NULL,
+ field_order = 'database',
+ show_in_getting_started = FALSE,
+ data_authority = 'unconfigured',
+ data_layer = 'internal',
+ visibility_type = CASE WHEN t.visibility_type = 'cruft' THEN 'cruft' ELSE NULL END
+ FROM metabase_table_user_settings AS s
+ WHERE s.table_id = t.id
+ - sql:
+ dbms: mysql,mariadb
+ sql: >-
+ UPDATE metabase_table t
+ JOIN metabase_table_user_settings s ON s.table_id = t.id
+ SET t.caveats = NULL,
+ t.points_of_interest = NULL,
+ t.owner_email = NULL,
+ t.owner_user_id = NULL,
+ t.is_published = FALSE,
+ t.collection_id = NULL,
+ t.field_order = 'database',
+ t.show_in_getting_started = FALSE,
+ t.data_authority = 'unconfigured',
+ t.data_layer = 'internal',
+ t.visibility_type = CASE WHEN t.visibility_type = 'cruft' THEN 'cruft' ELSE NULL END
+ - sql:
+ dbms: h2
+ sql: >-
+ UPDATE metabase_table
+ SET caveats = NULL,
+ points_of_interest = NULL,
+ owner_email = NULL,
+ owner_user_id = NULL,
+ is_published = FALSE,
+ collection_id = NULL,
+ field_order = 'database',
+ show_in_getting_started = FALSE,
+ data_authority = 'unconfigured',
+ data_layer = 'internal',
+ visibility_type = CASE WHEN visibility_type = 'cruft' THEN 'cruft' ELSE NULL END
+ WHERE EXISTS (SELECT 1 FROM metabase_table_user_settings s WHERE s.table_id = metabase_table.id)
+ rollback:
+ - sql:
+ dbms: postgresql
+ sql: >-
+ UPDATE metabase_table AS t
+ SET caveats = s.caveats,
+ points_of_interest = s.points_of_interest,
+ owner_email = s.owner_email,
+ owner_user_id = s.owner_user_id,
+ is_published = COALESCE(s.is_published, FALSE),
+ collection_id = s.collection_id,
+ field_order = COALESCE(s.field_order, 'database'),
+ show_in_getting_started = COALESCE(s.show_in_getting_started, FALSE),
+ data_authority = COALESCE(s.data_authority, 'unconfigured'),
+ data_layer = COALESCE(s.data_layer, 'internal'),
+ visibility_type = COALESCE(s.visibility_type, t.visibility_type)
+ FROM metabase_table_user_settings AS s
+ WHERE s.table_id = t.id
+ - sql:
+ dbms: mysql,mariadb
+ sql: >-
+ UPDATE metabase_table t
+ JOIN metabase_table_user_settings s ON s.table_id = t.id
+ SET t.caveats = s.caveats,
+ t.points_of_interest = s.points_of_interest,
+ t.owner_email = s.owner_email,
+ t.owner_user_id = s.owner_user_id,
+ t.is_published = COALESCE(s.is_published, FALSE),
+ t.collection_id = s.collection_id,
+ t.field_order = COALESCE(s.field_order, 'database'),
+ t.show_in_getting_started = COALESCE(s.show_in_getting_started, FALSE),
+ t.data_authority = COALESCE(s.data_authority, 'unconfigured'),
+ t.data_layer = COALESCE(s.data_layer, 'internal'),
+ t.visibility_type = COALESCE(s.visibility_type, t.visibility_type)
+ - sql:
+ dbms: h2
+ sql: >-
+ UPDATE metabase_table
+ SET caveats = (SELECT s.caveats FROM metabase_table_user_settings s WHERE s.table_id = metabase_table.id),
+ points_of_interest = (SELECT s.points_of_interest FROM metabase_table_user_settings s WHERE s.table_id = metabase_table.id),
+ owner_email = (SELECT s.owner_email FROM metabase_table_user_settings s WHERE s.table_id = metabase_table.id),
+ owner_user_id = (SELECT s.owner_user_id FROM metabase_table_user_settings s WHERE s.table_id = metabase_table.id),
+ is_published = COALESCE((SELECT s.is_published FROM metabase_table_user_settings s WHERE s.table_id = metabase_table.id), FALSE),
+ collection_id = (SELECT s.collection_id FROM metabase_table_user_settings s WHERE s.table_id = metabase_table.id),
+ field_order = COALESCE((SELECT s.field_order FROM metabase_table_user_settings s WHERE s.table_id = metabase_table.id), 'database'),
+ show_in_getting_started = COALESCE((SELECT s.show_in_getting_started FROM metabase_table_user_settings s WHERE s.table_id = metabase_table.id), FALSE),
+ data_authority = COALESCE((SELECT s.data_authority FROM metabase_table_user_settings s WHERE s.table_id = metabase_table.id), 'unconfigured'),
+ data_layer = COALESCE((SELECT s.data_layer FROM metabase_table_user_settings s WHERE s.table_id = metabase_table.id), 'internal'),
+ visibility_type = COALESCE((SELECT s.visibility_type FROM metabase_table_user_settings s WHERE s.table_id = metabase_table.id), visibility_type)
+ WHERE EXISTS (SELECT 1 FROM metabase_table_user_settings s WHERE s.table_id = metabase_table.id)
+
+ - changeSet:
+ id: v64.2026-09-11T00:00:07
+ author: ranquild
+ comment: Delete metabase_field_user_settings rows that hold no user value
+ changes:
+ - sql:
+ sql: >-
+ DELETE FROM metabase_field_user_settings
+ WHERE semantic_type IS NULL AND description IS NULL AND display_name IS NULL AND visibility_type IS NULL
+ AND fk_target_field_id IS NULL AND has_field_values IS NULL AND effective_type IS NULL
+ AND coercion_strategy IS NULL AND caveats IS NULL AND points_of_interest IS NULL AND nfc_path IS NULL
+ AND json_unfolding IS NULL AND settings IS NULL AND data_sensitivity IS NULL
+ AND description_set = FALSE AND semantic_type_set = FALSE AND fk_target_field_id_set = FALSE
+ rollback: # not necessary; the rows held nothing
+
+ - changeSet:
+ id: v64.2026-09-11T00:00:08
+ author: ranquild
+ comment: Add custom_position to metabase_field_user_settings
+ changes:
+ - addColumn:
+ tableName: metabase_field_user_settings
+ columns:
+ - column:
+ remarks: User-set custom_position for the Field
+ name: custom_position
+ type: int
+
+ - changeSet:
+ id: v64.2026-09-11T00:00:09
+ author: ranquild
+ comment: Create metabase_field_user_settings rows for the custom field positions in metabase_field
+ changes:
+ - sql:
+ sql: >-
+ INSERT INTO metabase_field_user_settings (field_id, custom_position)
+ SELECT f.id, f.custom_position
+ FROM metabase_field f
+ WHERE f.custom_position <> 0
+ AND NOT EXISTS (SELECT 1 FROM metabase_field_user_settings s WHERE s.field_id = f.id)
+ rollback: # not necessary; the positions are still in metabase_field
+
+ - changeSet:
+ id: v64.2026-09-11T00:00:10
+ author: ranquild
+ comment: Copy the custom field positions in metabase_field into existing metabase_field_user_settings rows
+ changes:
+ - sql:
+ dbms: postgresql
+ sql: >-
+ UPDATE metabase_field_user_settings AS s
+ SET custom_position = f.custom_position
+ FROM metabase_field AS f
+ WHERE f.id = s.field_id AND f.custom_position <> 0
+ - sql:
+ dbms: mysql,mariadb
+ sql: >-
+ UPDATE metabase_field_user_settings s
+ JOIN metabase_field f ON f.id = s.field_id
+ SET s.custom_position = f.custom_position
+ WHERE f.custom_position <> 0
+ - sql:
+ dbms: h2
+ sql: >-
+ UPDATE metabase_field_user_settings
+ SET custom_position = (SELECT f.custom_position FROM metabase_field f WHERE f.id = metabase_field_user_settings.field_id)
+ WHERE EXISTS (SELECT 1 FROM metabase_field f WHERE f.id = metabase_field_user_settings.field_id AND f.custom_position <> 0)
+ rollback: # not necessary; the positions are still in metabase_field
+
+ - changeSet:
+ id: v64.2026-09-11T00:00:11
+ author: ranquild
+ comment: Clear the custom field positions the backfill moved into metabase_field_user_settings
+ changes:
+ - sql:
+ sql: UPDATE metabase_field SET custom_position = 0 WHERE custom_position <> 0
+ rollback:
+ - sql:
+ dbms: postgresql
+ sql: >-
+ UPDATE metabase_field AS f
+ SET custom_position = s.custom_position
+ FROM metabase_field_user_settings AS s
+ WHERE s.field_id = f.id AND s.custom_position IS NOT NULL
+ - sql:
+ dbms: mysql,mariadb
+ sql: >-
+ UPDATE metabase_field f
+ JOIN metabase_field_user_settings s ON s.field_id = f.id
+ SET f.custom_position = s.custom_position
+ WHERE s.custom_position IS NOT NULL
+ - sql:
+ dbms: h2
+ sql: >-
+ UPDATE metabase_field
+ SET custom_position = (SELECT s.custom_position FROM metabase_field_user_settings s WHERE s.field_id = metabase_field.id)
+ WHERE EXISTS (SELECT 1 FROM metabase_field_user_settings s WHERE s.field_id = metabase_field.id AND s.custom_position IS NOT NULL)
diff --git a/resources/migrations/instance_analytics_views/fields/v3/h2-fields.sql b/resources/migrations/instance_analytics_views/fields/v3/h2-fields.sql
new file mode 100644
index 00000000000..cf285196f65
--- /dev/null
+++ b/resources/migrations/instance_analytics_views/fields/v3/h2-fields.sql
@@ -0,0 +1,19 @@
+drop view if exists v_fields;
+
+create or replace view v_fields as
+select
+ f.id as entity_id,
+ 'field_' || f.id as entity_qualified_id,
+ f.created_at,
+ f.updated_at,
+ f.name,
+ coalesce(u.display_name, f.display_name) as display_name,
+ case when u.description_set then u.description else f.description end as description,
+ f.base_type,
+ coalesce(u.visibility_type, f.visibility_type) as visibility_type,
+ case when u.fk_target_field_id_set then u.fk_target_field_id else f.fk_target_field_id end as fk_target_field_id,
+ coalesce(u.has_field_values, f.has_field_values) as has_field_values,
+ f.active,
+ f.table_id as table_id
+from metabase_field f
+left join metabase_field_user_settings u on u.field_id = f.id;
diff --git a/resources/migrations/instance_analytics_views/fields/v3/mysql-fields.sql b/resources/migrations/instance_analytics_views/fields/v3/mysql-fields.sql
new file mode 100644
index 00000000000..dfde1ee9838
--- /dev/null
+++ b/resources/migrations/instance_analytics_views/fields/v3/mysql-fields.sql
@@ -0,0 +1,21 @@
+drop view if exists v_fields;
+
+create or replace
+SQL SECURITY INVOKER
+view v_fields as
+select
+ f.id as entity_id,
+ concat('field_', f.id) as entity_qualified_id, -- should match idx_field_entity_qualified_id
+ f.created_at,
+ f.updated_at,
+ f.name,
+ coalesce(u.display_name, f.display_name) as display_name,
+ case when u.description_set then u.description else f.description end as description,
+ f.base_type,
+ coalesce(u.visibility_type, f.visibility_type) as visibility_type,
+ case when u.fk_target_field_id_set then u.fk_target_field_id else f.fk_target_field_id end as fk_target_field_id,
+ coalesce(u.has_field_values, f.has_field_values) as has_field_values,
+ f.active,
+ f.table_id as table_id
+from metabase_field f
+left join metabase_field_user_settings u on u.field_id = f.id;
diff --git a/resources/migrations/instance_analytics_views/fields/v3/postgres-fields.sql b/resources/migrations/instance_analytics_views/fields/v3/postgres-fields.sql
new file mode 100644
index 00000000000..a566ae805ca
--- /dev/null
+++ b/resources/migrations/instance_analytics_views/fields/v3/postgres-fields.sql
@@ -0,0 +1,19 @@
+drop view if exists v_fields;
+
+create or replace view v_fields as
+select
+ f.id as entity_id,
+ 'field_' || f.id as entity_qualified_id, -- should match idx_field_entity_qualified_id
+ f.created_at,
+ f.updated_at,
+ f.name,
+ coalesce(u.display_name, f.display_name) as display_name,
+ case when u.description_set then u.description else f.description end as description,
+ f.base_type,
+ coalesce(u.visibility_type, f.visibility_type) as visibility_type,
+ case when u.fk_target_field_id_set then u.fk_target_field_id else f.fk_target_field_id end as fk_target_field_id,
+ coalesce(u.has_field_values, f.has_field_values) as has_field_values,
+ f.active,
+ f.table_id as table_id
+from metabase_field f
+left join metabase_field_user_settings u on u.field_id = f.id;
diff --git a/src/metabase/actions/db.clj b/src/metabase/actions/db.clj
index 440645bb0c7..c56501971a9 100644
--- a/src/metabase/actions/db.clj
+++ b/src/metabase/actions/db.clj
@@ -7,6 +7,7 @@
[metabase.queries.schema :as queries.schema]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
(mu/defn database-for-action
@@ -21,7 +22,7 @@
(mu/defn table-database-id
"The Database id of the Table with `table-id`, or nil."
[table-id :- ::lib.schema.id/table]
- (t2/select-one-fn :db_id [:model/Table :db_id] table-id))
+ (t2/select-one-fn :db_id [:model/Table :db_id] :id table-id {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))
(mu/defn card-query
"The query of the Card with `card-id`, or nil."
@@ -51,12 +52,12 @@
(mu/defn table
"The Table with `table-id`, or nil."
[table-id :- ::lib.schema.id/table]
- (t2/select-one :model/Table :id table-id))
+ (t2/select-one :model/Table :id table-id {:from [(warehouse-schema-overlay/table-query)]}))
(mu/defn tables
"The Tables with `table-ids`."
[table-ids :- [:sequential ::lib.schema.id/table]]
- (t2/select :model/Table :id [:in table-ids]))
+ (t2/select :model/Table :id [:in table-ids] {:from [(warehouse-schema-overlay/table-query)]}))
(mu/defn database
"The Database with `database-id`, or nil."
@@ -211,7 +212,7 @@
(mu/defn fields-for-parameters
"The id, base type, display name, and description of the Fields with `field-ids`."
[field-ids :- [:set ::lib.schema.id/field]]
- (t2/select [:model/Field :id :base_type :display_name :description] :id [:in field-ids]))
+ (t2/select [:model/Field :id :base_type :display_name :description] :id [:in field-ids] {:from [(warehouse-schema-overlay/field-query)]}))
(mu/defn action-database-settings
"The id and Database settings of the Actions with `action-ids`."
@@ -240,9 +241,9 @@
(mu/defn writable-table-exists?
"Whether the Database with `database-id` has a writable Table."
[database-id :- ::lib.schema.id/database]
- (t2/exists? :model/Table :db_id database-id :is_writable true))
+ (t2/exists? :model/Table :db_id database-id :is_writable true {:from [(warehouse-schema-overlay/table-query)]}))
(mu/defn table-with-unknown-writability-exists?
"Whether the Database with `database-id` has a Table whose writability is unknown."
[database-id :- ::lib.schema.id/database]
- (t2/exists? :model/Table :db_id database-id :is_writable nil))
+ (t2/exists? :model/Table :db_id database-id :is_writable nil {:from [(warehouse-schema-overlay/table-query)]}))
diff --git a/src/metabase/activity_feed/db.clj b/src/metabase/activity_feed/db.clj
index 478f412c9b8..ceec800c178 100644
--- a/src/metabase/activity_feed/db.clj
+++ b/src/metabase/activity_feed/db.clj
@@ -8,6 +8,7 @@
[metabase.util.honey-sql-2 :as h2x]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
(mu/defn recent-cards
@@ -40,7 +41,8 @@
:display_name [:metabase_database.initial_sync_status :initial-sync-status]
[:visibility_type :visibility_type]
[:metabase_database.name :database-name]]
- {:where [:in :metabase_table.id ids]
+ {:from [(warehouse-schema-overlay/table-query)]
+ :where [:in :metabase_table.id ids]
:left-join [:metabase_database [:= :metabase_database.id :metabase_table.db_id]]}))
(mu/defn recent-dashboard-and-table-views
@@ -65,7 +67,7 @@
[:and
[:= :model "dashboard"]
[:= :d.id :model_id]]
- [:metabase_table :t]
+ (warehouse-schema-overlay/table-query {:alias :t})
[:and
[:= :model "table"]
[:= :t.id :model_id]]]}))
@@ -97,7 +99,7 @@
(mu/defn table-exists?
"Whether a Table with `id` exists."
[id :- ::lib.schema.id/table]
- (t2/exists? :model/Table :id id))
+ (t2/exists? :model/Table :id id {:from [(warehouse-schema-overlay/table-query)]}))
(mu/defn collection-exists?
"Whether a Collection with `id` exists."
@@ -117,7 +119,7 @@
(mu/defn table
"The Table with `id`, or nil."
[id :- ::lib.schema.id/table]
- (t2/select-one :model/Table :id id))
+ (t2/select-one :model/Table :id id {:from [(warehouse-schema-overlay/table-query)]}))
(mu/defn collection
"The Collection with `id`, or nil."
@@ -278,7 +280,7 @@
[:db.name :database-name]
[:db.id :db_id]
[:db.initial_sync_status :initial-sync-status]]
- :from [[:metabase_table :t]]
+ :from [(warehouse-schema-overlay/table-query {:alias :t})]
:where [:and
[:or
[:= :visibility_type nil]
diff --git a/src/metabase/analytics/db.clj b/src/metabase/analytics/db.clj
index 31ca075ecdc..aaf2848d258 100644
--- a/src/metabase/analytics/db.clj
+++ b/src/metabase/analytics/db.clj
@@ -9,6 +9,7 @@
[metabase.util.honey-sql-2 :as h2x]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
(mu/defn first-user-date-joined
@@ -49,7 +50,8 @@
(mu/defn published-table-count-in-collections
"The number of published Tables in the Collections with `collection-ids`."
[collection-ids :- [:set ::lib.schema.id/collection]]
- (t2/count :model/Table {:where [:and
+ (t2/count :model/Table {:from [(warehouse-schema-overlay/table-query)]
+ :where [:and
[:= :is_published true]
[:in :collection_id collection-ids]]}))
@@ -161,7 +163,7 @@
"The Database id and schema of the Tables of the non-internal Databases."
[]
(t2/query {:select [:t.db_id :t.schema]
- :from [[(t2/table-name :model/Table) :t]]
+ :from [(warehouse-schema-overlay/table-query {:alias :t, :user-settings? false})]
:join [[(t2/table-name :model/Database) :d] [:= :d.id :t.db_id]]
:where (mi/exclude-internal-content-hsql :model/Database :table-alias :d)}))
@@ -169,8 +171,8 @@
"The Table id of the Fields of the non-internal Databases."
[]
(t2/query {:select [:f.table_id]
- :from [[(t2/table-name :model/Field) :f]]
- :join [[(t2/table-name :model/Table) :t] [:= :t.id :f.table_id]
+ :from [(warehouse-schema-overlay/field-query {:alias :f})]
+ :join [(warehouse-schema-overlay/table-query {:alias :t}) [:= :t.id :f.table_id]
[(t2/table-name :model/Database) :d] [:= :d.id :t.db_id]]
:where (mi/exclude-internal-content-hsql :model/Database :table-alias :d)}))
@@ -373,7 +375,7 @@
(mu/defn upload-table-exists?
"Whether an uploaded Table exists."
[]
- (t2/exists? :model/Table :is_upload true))
+ (t2/exists? :model/Table :is_upload true {:from [(warehouse-schema-overlay/table-query)]}))
(mu/defn snippet-collection-exists?
"Whether a snippet Collection exists."
diff --git a/src/metabase/analytics/stats.clj b/src/metabase/analytics/stats.clj
index 30ed326851e..acf162fe496 100644
--- a/src/metabase/analytics/stats.clj
+++ b/src/metabase/analytics/stats.clj
@@ -18,7 +18,6 @@
[metabase.eid-translation.core :as eid-translation]
[metabase.internal-stats.core :as internal-stats]
[metabase.lib-be.core :as lib-be]
- [metabase.models.humanization :as humanization]
[metabase.premium-features.core :as premium-features :refer [defenterprise]]
[metabase.session.settings :as session.settings]
[metabase.settings.core :as setting]
@@ -124,7 +123,7 @@
:check_for_updates (version/check-for-updates)
:report_timezone (driver/report-timezone)
;; We deprecated advanced humanization but have this here anyways
- :friendly_names (= (humanization/humanization-strategy) "advanced")
+ :friendly_names (= (setting/get :humanization-strategy) "advanced")
:email_configured (setting/get :email-configured?)
:slack_configured (setting/get :slack-configured?)
:sso_configured (setting/get :google-auth-enabled)
diff --git a/src/metabase/audit_app/db.clj b/src/metabase/audit_app/db.clj
index b040178a869..c6ff7f815e1 100644
--- a/src/metabase/audit_app/db.clj
+++ b/src/metabase/audit_app/db.clj
@@ -6,6 +6,7 @@
[metabase.lib.schema.id :as lib.schema.id]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
(mu/defn cards
@@ -36,7 +37,7 @@
(mu/defn table-database-id
"The Database id of the Table with `table-id`, or nil."
[table-id :- ::lib.schema.id/table]
- (t2/select-one-fn :db_id :model/Table, :id table-id))
+ (t2/select-one-fn :db_id :model/Table, :id table-id {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))
(mu/defn insert-audit-log!
"Insert an AuditLog entry."
diff --git a/src/metabase/cmd/copy.clj b/src/metabase/cmd/copy.clj
index 656f57dc854..834315840cd 100644
--- a/src/metabase/cmd/copy.clj
+++ b/src/metabase/cmd/copy.clj
@@ -65,6 +65,7 @@
:model/Field
:model/FieldValues
:model/FieldUserSettings
+ :model/TableUserSettings
:model/Segment
:model/ModerationReview
:model/Revision
@@ -409,6 +410,7 @@
:model/ImplicitAction
:model/HTTPAction
:model/FieldUserSettings
+ :model/TableUserSettings
:model/QueryAction
:model/MetabotConversation
:model/ModelIndexValue
diff --git a/src/metabase/collections/children.clj b/src/metabase/collections/children.clj
index 257c18c4d0c..5ec3007872e 100644
--- a/src/metabase/collections/children.clj
+++ b/src/metabase/collections/children.clj
@@ -37,6 +37,7 @@
[metabase.util :as u]
[metabase.util.honey-sql-2 :as h2x]
[metabase.util.malli :as mu]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
@@ -763,7 +764,7 @@
[:t.db_id :database_id]
[[:!= :t.archived_at nil] :archived]
[(h2x/literal "table") :model]]
- :from [[:metabase_table :t]]
+ :from [(warehouse-schema-overlay/table-query {:alias :t})]
:where [:and
[:= :t.is_published true]
(poison-when-pinned-clause pinned-state)
diff --git a/src/metabase/collections/db.clj b/src/metabase/collections/db.clj
index 2972058b2cb..977b832995e 100644
--- a/src/metabase/collections/db.clj
+++ b/src/metabase/collections/db.clj
@@ -10,6 +10,7 @@
[metabase.models.serialization :as serdes]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
(def ^:private PermissionsRow
@@ -498,7 +499,8 @@
`skip-archived?`."
[collection-id :- [:maybe ::lib.schema.id/collection]
skip-archived? :- [:maybe :boolean]]
- (t2/select-pks-set :model/Table {:where [:and
+ (t2/select-pks-set :model/Table {:from [(warehouse-schema-overlay/table-query)]
+ :where [:and
[:= :collection_id collection-id]
[:= :is_published true]
(when skip-archived? [:= :archived_at nil])]}))
@@ -511,13 +513,18 @@
(mu/defn published-table-ids-in-collections
"The IDs of the published Tables in the Collections with `collection-ids`."
[collection-ids :- [:or [:set ::lib.schema.id/collection] [:sequential ::lib.schema.id/collection]]]
- (t2/select-pks-set :model/Table :collection_id [:in collection-ids] :is_published true))
+ (t2/select-pks-set :model/Table :collection_id [:in collection-ids] :is_published true {:from [(warehouse-schema-overlay/table-query)]}))
(mu/defn unpublish-tables-in-collections!
- "Unpublish the Tables in the Collections with `collection-ids` and detach them from their ::collections.schema/collection, returning
- the number updated."
+ "Unpublish the Tables in the Collections with `collection-ids`, in `metabase_table` and in their user settings,
+ returning the number updated."
[collection-ids :- [:or [:set ::lib.schema.id/collection] [:sequential ::lib.schema.id/collection]]]
- (t2/update! :model/Table {:collection_id [:in collection-ids]} {:collection_id nil, :is_published false}))
+ (let [table-ids (published-table-ids-in-collections collection-ids)]
+ (when (seq table-ids)
+ (t2/update! :model/TableUserSettings :table_id [:in table-ids]
+ {:collection_id nil, :is_published false}))
+ (t2/update! :model/Table {:collection_id [:in collection-ids]}
+ {:collection_id nil, :is_published false})))
(mu/defn dashboard-ids-with-cards
"The `:dashboard_id` rows of the Dashboards among `dashboard-ids` holding an unarchived dashboard question."
@@ -549,7 +556,7 @@
"The distinct `:collection_id`s of the published, unarchived Tables in the Collections with `collection-ids`."
[collection-ids]
(t2/query {:select-distinct [:collection_id]
- :from :metabase_table
+ :from [(warehouse-schema-overlay/table-query)]
:where [:and
[:= :is_published true]
[:= :archived_at nil]
diff --git a/src/metabase/collections_rest/db.clj b/src/metabase/collections_rest/db.clj
index b0c0626cff0..080ac511427 100644
--- a/src/metabase/collections_rest/db.clj
+++ b/src/metabase/collections_rest/db.clj
@@ -4,6 +4,7 @@
(:require
[metabase.lib.schema.id :as lib.schema.id]
[metabase.util.malli :as mu]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
(mu/defn collection
@@ -32,10 +33,10 @@
"The distinct `:collection_id`s of the published, unarchived Tables."
[]
(t2/query {:select-distinct [:collection_id]
- :from :metabase_table
- :where [:and
- [:= :is_published true]
- [:= :archived_at nil]]}))
+ :from [(warehouse-schema-overlay/table-query)]
+ :where [:and
+ [:= :is_published true]
+ [:= :archived_at nil]]}))
(mu/defn top-level-cards-in-collection
"The Cards in the Collection with `collection-id` that belong to no Dashboard, newest first."
diff --git a/src/metabase/core/init.clj b/src/metabase/core/init.clj
index d9ca14f8cff..9206ae30ec2 100644
--- a/src/metabase/core/init.clj
+++ b/src/metabase/core/init.clj
@@ -87,6 +87,7 @@
[metabase.util.experiment.init]
[metabase.version.init]
[metabase.view-log.init]
+ [metabase.warehouse-schema.init]
[metabase.warehouses.init]
[metabase.xrays.init]))
diff --git a/src/metabase/data_studio/api/table.clj b/src/metabase/data_studio/api/table.clj
index 34429a4e1af..71a08eabb4f 100644
--- a/src/metabase/data_studio/api/table.clj
+++ b/src/metabase/data_studio/api/table.clj
@@ -18,7 +18,8 @@
[metabase.util.malli.registry :as mr]
[metabase.util.malli.schema :as ms]
[metabase.util.quick-task :as quick-task]
- [metabase.warehouse-schema.models.table :as table]))
+ [metabase.warehouse-schema.models.table :as table]
+ [metabase.warehouse-schema.models.table-user-settings :as schema.table-user-settings]))
(set! *warn-on-reflection* true)
@@ -177,7 +178,7 @@
table-ids (set (map :id existing-tables))
set-map (select-keys body set-ks)]
(when (seq set-map)
- (data-studio.db/update-tables! table-ids set-map)
+ (schema.table-user-settings/upsert-user-settings-for-tables! table-ids set-map)
(maybe-sync-unhidden-tables! existing-tables set-map)
;; Publish update events for remote sync tracking
(let [updated-tables (data-studio.db/tables table-ids)]
diff --git a/src/metabase/data_studio/db.clj b/src/metabase/data_studio/db.clj
index 362a935f2de..ce2fb0773bb 100644
--- a/src/metabase/data_studio/db.clj
+++ b/src/metabase/data_studio/db.clj
@@ -3,11 +3,10 @@
additional logic, so the rest of the module never talks to `toucan2.core` itself."
(:require
[clojure.string :as str]
- [malli.util :as mut]
[metabase.lib.schema.id :as lib.schema.id]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
- [metabase.warehouse-schema.schema :as warehouse-schema.schema]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
(def ^:private TableSelectors
@@ -39,9 +38,9 @@
exclude-table-ids :- [:set ::lib.schema.id/table]]
(t2/reducible-query {:select [[output-table-id :table_id]]
:from [[(t2/table-name :model/Dimension) :dim]]
- :join [[(t2/table-name :model/Field) :source_field]
+ :join [(warehouse-schema-overlay/field-query {:alias :source_field, :user-settings? false})
[:= :dim.field_id :source_field.id]
- [(t2/table-name :model/Field) :target_field]
+ (warehouse-schema-overlay/field-query {:alias :target_field, :user-settings? false})
[:= :dim.human_readable_field_id :target_field.id]]
:where [:and
[:= :dim.type "external"]
@@ -58,17 +57,17 @@
(let [where (table-selectors-where selectors)]
(t2/reducible-query {:select [[output-table-id :table_id]]
:from [[(t2/table-name :model/Dimension) :dim]]
- :join [[(t2/table-name :model/Field) :source_field]
+ :join [(warehouse-schema-overlay/field-query {:alias :source_field})
[:= :dim.field_id :source_field.id]
- [(t2/table-name :model/Field) :target_field]
+ (warehouse-schema-overlay/field-query {:alias :target_field})
[:= :dim.human_readable_field_id :target_field.id]]
:where [:and
[:= :dim.type "external"]
[:in input-table-id ^:allow-subquery
- {:select [:id] :from [(t2/table-name :model/Table)] :where where}]
+ {:select [:id] :from [(warehouse-schema-overlay/table-query)] :where where}]
[:not [:exists ^:allow-subquery
{:select [1]
- :from [(t2/table-name :model/Table)]
+ :from [(warehouse-schema-overlay/table-query)]
:where [:and where [:= :id output-table-id]]}]]]})))
(mu/defn database
@@ -84,35 +83,32 @@
(mu/defn tables-matching-selectors
"The Tables picked out by `selectors`, a map of `:database_ids`, `:schema_ids`, and/or `:table_ids`."
[selectors :- TableSelectors]
- (t2/select :model/Table {:where (table-selectors-where selectors)}))
+ (t2/select :model/Table {:from [(warehouse-schema-overlay/table-query)]
+ :where (table-selectors-where selectors)}))
(mu/defn tables-matching-selectors-in-id-order
"The Tables picked out by `selectors`, in id order."
[selectors :- TableSelectors]
- (t2/select :model/Table {:where (table-selectors-where selectors), :order-by [[:id]]}))
+ (t2/select :model/Table {:from [(warehouse-schema-overlay/table-query)]
+ :where (table-selectors-where selectors), :order-by [[:id]]}))
(mu/defn tables
"The Tables with `table-ids`."
[table-ids :- [:set ::lib.schema.id/table]]
- (t2/select :model/Table :id [:in table-ids]))
-
-(mu/defn update-tables!
- "Apply `changes` to the Tables with `table-ids`, returning the number updated."
- [table-ids :- [:set ::lib.schema.id/table]
- changes :- (mut/select-keys ::warehouse-schema.schema/table.update [:data_authority :data_source :data_layer :entity_type :owner_email :owner_user_id])]
- (t2/update! :model/Table [:in table-ids] changes))
+ (t2/select :model/Table :id [:in table-ids] {:from [(warehouse-schema-overlay/table-query)]}))
(mu/defn selection-columns-for-selectors
"Up to `limit` id, database, name, schema, and published flag of the Tables picked out by `selectors`."
[selectors :- TableSelectors
limit :- ms/PositiveInt]
(t2/select [:model/Table :id :db_id :name :display_name :schema :is_published]
- {:where (table-selectors-where selectors), :limit limit}))
+ {:from [(warehouse-schema-overlay/table-query)]
+ :where (table-selectors-where selectors), :limit limit}))
(mu/defn selection-columns-for-tables
"The id, database, name, schema, and published flag of the Tables with `table-ids`."
[table-ids :- [:set ::lib.schema.id/table]]
- (t2/select [:model/Table :id :db_id :name :display_name :schema :is_published] :id [:in table-ids]))
+ (t2/select [:model/Table :id :db_id :name :display_name :schema :is_published] :id [:in table-ids] {:from [(warehouse-schema-overlay/table-query)]}))
(mu/defn delete-field-values-for-tables!
"Delete the FieldValues of the Fields of the Tables with `table-ids`, returning the number deleted."
diff --git a/src/metabase/documents/db.clj b/src/metabase/documents/db.clj
index 16511ee7cd1..4770d332f1b 100644
--- a/src/metabase/documents/db.clj
+++ b/src/metabase/documents/db.clj
@@ -10,6 +10,7 @@
[metabase.queries.schema :as queries.schema]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
(mu/defn document
@@ -141,7 +142,7 @@
(mu/defn table
"The Table with `id`, or nil."
[id :- ::lib.schema.id/table]
- (t2/select-one :model/Table :id id))
+ (t2/select-one :model/Table :id id {:from [(warehouse-schema-overlay/table-query)]}))
(mu/defn dashboard
"The Dashboard with `id`, or nil."
diff --git a/src/metabase/driver/db.clj b/src/metabase/driver/db.clj
index 57e899abcdc..88d8056e606 100644
--- a/src/metabase/driver/db.clj
+++ b/src/metabase/driver/db.clj
@@ -4,6 +4,7 @@
(:require
[metabase.lib.schema.id :as lib.schema.id]
[metabase.util.malli :as mu]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
^{:clj-kondo/ignore [:discouraged-namespace]}
[toucan2.core :as t2]))
@@ -15,7 +16,7 @@
(mu/defn table-field-names
"The names of the Fields of the Table with `table-id`."
[table-id :- ::lib.schema.id/table]
- (t2/select-fn-vec :name [:model/Field :name] :table_id table-id))
+ (t2/select-fn-vec :name [:model/Field :name] :table_id table-id {:from [(warehouse-schema-overlay/field-query {:user-settings? false})]}))
(mu/defn database-connection-details
"The engine and connection details of the Database with `database-id`, or nil."
@@ -28,4 +29,5 @@
(t2/select-fn-set :name [:model/Field :name]
:table_id table-id
:base_type :type/JSON
- :json_unfolding false))
+ :json_unfolding false
+ {:from [(warehouse-schema-overlay/field-query)]}))
diff --git a/src/metabase/embedding_hub/db.clj b/src/metabase/embedding_hub/db.clj
index eea089e8b9c..fe0b4ab0665 100644
--- a/src/metabase/embedding_hub/db.clj
+++ b/src/metabase/embedding_hub/db.clj
@@ -2,6 +2,7 @@
"Application database queries for the embedding hub module. Every function here is a direct Toucan 2 call with no
additional logic, so the rest of the module never talks to `toucan2.core` itself."
(:require
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
(defn user-added-database?
@@ -19,7 +20,8 @@
(defn uploaded-table?
"Whether the database with `database-id` has an active uploaded table."
[database-id]
- (t2/exists? :model/Table {:where [:and
+ (t2/exists? :model/Table {:from [(warehouse-schema-overlay/table-query)]
+ :where [:and
[:= :active true]
[:= :is_upload true]
[:= :db_id database-id]]}))
diff --git a/src/metabase/lib_be/metadata/jvm.clj b/src/metabase/lib_be/metadata/jvm.clj
index 5f19695e06a..b0fedaf5f88 100644
--- a/src/metabase/lib_be/metadata/jvm.clj
+++ b/src/metabase/lib_be/metadata/jvm.clj
@@ -22,6 +22,7 @@
[metabase.util.memoize :as u.memo]
[metabase.util.performance :as perf :refer [get-in]]
[metabase.util.snake-hating-map :as u.snake-hating-map]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[methodical.core :as methodical]
[potemkin :as p]
[pretty.core :as pretty]
@@ -113,7 +114,8 @@
#_resolved-query clojure.lang.IPersistentMap]
[query-type model parsed-args honeysql]
(merge (next-method query-type model parsed-args honeysql)
- {:select [:id :db_id :name :display_name :schema :active :visibility_type :database_require_filter]}))
+ {:select [:id :db_id :name :display_name :schema :active :visibility_type :database_require_filter]
+ :from [(warehouse-schema-overlay/table-query)]}))
(t2/define-after-select :metadata/table
[table]
@@ -182,7 +184,7 @@
:dimension/type
:values/human_readable_values
:values/values]
- :from [[(t2/table-name :model/Field) :field]]
+ :from [(warehouse-schema-overlay/field-query {:alias :field})]
:left-join [[(t2/table-name :model/Table) :table]
[:= :field/table_id :table/id]
[(t2/table-name :model/Dimension) :dimension]
diff --git a/src/metabase/lib_metric/db.clj b/src/metabase/lib_metric/db.clj
index 1294dc76eb4..4be478cedc5 100644
--- a/src/metabase/lib_metric/db.clj
+++ b/src/metabase/lib_metric/db.clj
@@ -6,12 +6,13 @@
[metabase.lib.schema.id :as lib.schema.id]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
(mu/defn fields
"The Fields with `field-ids`."
[field-ids :- [:set ::lib.schema.id/field]]
- (t2/select :model/Field :id [:in field-ids]))
+ (t2/select :model/Field :id [:in field-ids] {:from [(warehouse-schema-overlay/field-query)]}))
(mu/defn card-table-and-database-id
"The `:table_id` and `:database_id` of the Card with `card-id`, or nil."
@@ -21,7 +22,7 @@
(mu/defn table-database-id
"The Database id of the Table with `table-id`, or nil."
[table-id :- ::lib.schema.id/table]
- (t2/select-one-fn :db_id :model/Table table-id))
+ (t2/select-one-fn :db_id :model/Table :id table-id {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))
(defn- metric-where
"The `:where` map for the metric Cards picked out by `metadata-spec` (its `:id`, `:name`,
diff --git a/src/metabase/llm/db.clj b/src/metabase/llm/db.clj
index 0e9b67764fa..9d3fc7616cf 100644
--- a/src/metabase/llm/db.clj
+++ b/src/metabase/llm/db.clj
@@ -6,6 +6,7 @@
[metabase.models.interface :as mi]
[metabase.util :as u]
[metabase.util.malli :as mu]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
(mu/defn database-engine
@@ -31,7 +32,8 @@
[:table :string]
[:schema {:optional true} [:maybe :string]]]]]
(t2/select :model/Table
- {:where [:and
+ {:from [(warehouse-schema-overlay/table-query)]
+ :where [:and
[:= :db_id database-id]
[:= :active true]
(into [:or] (map table-match-clause) tables)]}))
@@ -54,7 +56,8 @@
:db_id database-id
:active true
:visibility_type nil
- (cond-> {:where clause}
+ (cond-> {:from [(warehouse-schema-overlay/table-query)]
+ :where clause}
with (assoc :with with)))))
(mu/defn unarchived-cards
@@ -65,14 +68,14 @@
(mu/defn fields
"The Fields with `field-ids`."
[field-ids :- [:set ::lib.schema.id/field]]
- (t2/select :model/Field :id [:in field-ids]))
+ (t2/select :model/Field :id [:in field-ids] {:from [(warehouse-schema-overlay/field-query)]}))
(mu/defn field-names-and-tables
"The id, name, and Table id of the Fields with `field-ids`."
[field-ids :- [:set ::lib.schema.id/field]]
- (t2/select [:model/Field :id :name :table_id] :id [:in field-ids]))
+ (t2/select [:model/Field :id :name :table_id] :id [:in field-ids] {:from [(warehouse-schema-overlay/field-query {:user-settings? false})]}))
(mu/defn field-fingerprints
"A map of Field id to fingerprint for the Fields with `field-ids`."
[field-ids :- [:set ::lib.schema.id/field]]
- (t2/select-pk->fn :fingerprint :model/Field :id [:in field-ids]))
+ (t2/select-pk->fn :fingerprint :model/Field :id [:in field-ids] {:from [(warehouse-schema-overlay/field-query {:user-settings? false})]}))
diff --git a/src/metabase/mcp/db.clj b/src/metabase/mcp/db.clj
index 013ba17e3ea..dcb03bd24e6 100644
--- a/src/metabase/mcp/db.clj
+++ b/src/metabase/mcp/db.clj
@@ -12,6 +12,7 @@
[metabase.session.core :as session]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
(defn select-one-by-id
@@ -39,12 +40,12 @@
"The Table with `table-id`, or nil. Unlike [[select-one-by-id]], a nil `table-id` is answered with nil
rather than refused — callers reach here with a column value, not with an id an agent supplied."
[table-id]
- (t2/select-one :model/Table :id table-id))
+ (t2/select-one :model/Table :id table-id {:from [(warehouse-schema-overlay/table-query)]}))
(defn field-types
"The `base_type` and `effective_type` of the Field with `field-id`, or nil."
[field-id]
- (t2/select-one [:model/Field :base_type :effective_type] :id field-id))
+ (t2/select-one [:model/Field :base_type :effective_type] :id field-id {:from [(warehouse-schema-overlay/field-query)]}))
(defn nullable-fields-with-fingerprints
"The Fields among `field-ids` the warehouse declares nullable, carrying their `:fingerprint`.
@@ -53,7 +54,8 @@
[field-ids]
(t2/select [:model/Field :id :database_is_nullable :fingerprint]
:id [:in field-ids]
- :database_is_nullable true))
+ :database_is_nullable true
+ {:from [(warehouse-schema-overlay/field-query)]}))
(defn active-user-exists?
"Whether an active User with `user-id` exists."
@@ -97,7 +99,7 @@
"The active Tables with `table-ids`. `table-ids` is expected non-empty — an empty `:in` is a SQL error
rather than an empty result, so callers guard it."
[table-ids]
- (t2/select :model/Table :id [:in table-ids] :active true))
+ (t2/select :model/Table :id [:in table-ids] :active true {:from [(warehouse-schema-overlay/table-query)]}))
(defn active-visible-fields-for-tables
"The id, name, table id, and position of the active, non-hidden Fields of the Tables with `table-ids`, in
@@ -108,7 +110,8 @@
:table_id [:in table-ids]
:active true
:visibility_type [:not-in ["hidden" "sensitive" "retired"]]
- {:order-by [[:position :asc] [:id :asc]]}))
+ {:from [(warehouse-schema-overlay/field-query)]
+ :order-by [[:position :asc] [:id :asc]]}))
(defn collections-for-read-check
"The Collections with `ids`, carrying the columns [[metabase.models.interface/can-read?]] consults.
diff --git a/src/metabase/measures/db.clj b/src/metabase/measures/db.clj
index 7edf280d87f..dfc2258a4ae 100644
--- a/src/metabase/measures/db.clj
+++ b/src/metabase/measures/db.clj
@@ -6,6 +6,7 @@
[metabase.lib.schema.id :as lib.schema.id]
[metabase.measures.schema :as measures.schema]
[metabase.util.malli :as mu]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
(mu/defn insert-measure!
@@ -33,7 +34,7 @@
(mu/defn table-database-ids
"The set of Database ids of the Tables with `table-ids`."
[table-ids :- [:set ::lib.schema.id/table]]
- (t2/select-fn-set :db_id :model/Table :id [:in table-ids]))
+ (t2/select-fn-set :db_id :model/Table :id [:in table-ids] {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))
(mu/defn update-measure!
"Apply `changes` to the Measure with `id`."
@@ -44,12 +45,12 @@
(mu/defn table
"The Table with `table-id`, or nil."
[table-id :- ::lib.schema.id/table]
- (t2/select-one :model/Table :id table-id))
+ (t2/select-one :model/Table :id table-id {:from [(warehouse-schema-overlay/table-query)]}))
(mu/defn table-perms-columns
"The Database id, schema, and id of the Table with `table-id`, or nil."
[table-id :- ::lib.schema.id/table]
- (t2/select-one [:model/Table :db_id :schema :id] :id table-id))
+ (t2/select-one [:model/Table :db_id :schema :id] :id table-id {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))
(mu/defn collections
"The Collections with `collection-ids`."
diff --git a/src/metabase/metabot/db.clj b/src/metabase/metabot/db.clj
index e372d9ba4bf..324e0451281 100644
--- a/src/metabase/metabot/db.clj
+++ b/src/metabase/metabot/db.clj
@@ -16,6 +16,7 @@
[metabase.util :as u]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
(declare collection metabot-metrics-and-models-query root-collections-of-types)
@@ -431,7 +432,7 @@
(mu/defn table
"The Table with `table-id`, or nil."
[table-id :- [:maybe ::lib.schema.id/table]]
- (t2/select-one :model/Table :id table-id))
+ (t2/select-one :model/Table :id table-id {:from [(warehouse-schema-overlay/table-query)]}))
(mu/defn active-table-with-columns
"The `columns` of the active Table with `table-id`, or nil."
@@ -442,7 +443,7 @@
(mu/defn table-database-id
"The Database ID of the Table with `table-id`."
[table-id :- ::lib.schema.id/table]
- (t2/select-one-fn :db_id :model/Table :id table-id))
+ (t2/select-one-fn :db_id :model/Table :id table-id {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))
(defn readable-active-table-database-id
"The Database ID of the active Table with `table-id`, or nil when there is no such table *or*
@@ -458,29 +459,29 @@
and `:collection_id`. Handing it a trimmed row makes it answer a narrower question and deny a
user whose access comes through that path."
[table-id]
- (when-let [table (t2/select-one :model/Table :id table-id :active true)]
+ (when-let [table (t2/select-one :model/Table :id table-id :active true {:from [(warehouse-schema-overlay/table-query)]})]
(when (mi/can-read? table)
(:db_id table))))
(mu/defn tables-by-id
"A map of ID to Table for `table-ids`."
[table-ids :- [:set ::lib.schema.id/table]]
- (t2/select-fn->fn :id identity :model/Table :id [:in table-ids]))
+ (t2/select-fn->fn :id identity :model/Table :id [:in table-ids] {:from [(warehouse-schema-overlay/table-query)]}))
(mu/defn table-summaries
"The ID, names, schema, Database ID, and description of the Tables with `table-ids`."
[table-ids :- [:sequential ::lib.schema.id/table]]
- (t2/select [:model/Table :id :name :display_name :schema :db_id :description] :id [:in table-ids]))
+ (t2/select [:model/Table :id :name :display_name :schema :db_id :description] :id [:in table-ids] {:from [(warehouse-schema-overlay/table-query)]}))
(mu/defn table-schema-rows
"The ID, name, schema, and Database ID of the Tables with `table-ids`."
[table-ids :- [:sequential ::lib.schema.id/table]]
- (t2/select [:model/Table :id :name :schema :db_id] :id [:in table-ids]))
+ (t2/select [:model/Table :id :name :schema :db_id] :id [:in table-ids] {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))
(mu/defn table-curation-rows
"The ID, published flag, data layer, and data authority of the Tables with `table-ids`."
[table-ids :- [:sequential ::lib.schema.id/table]]
- (t2/select [:model/Table :id :is_published :data_layer :data_authority] :id [:in table-ids]))
+ (t2/select [:model/Table :id :is_published :data_layer :data_authority] :id [:in table-ids] {:from [(warehouse-schema-overlay/table-query)]}))
(mu/defn visible-table-summaries
"The ID, name, schema, and description of the active, unhidden Tables among `table-ids` in the Database with
@@ -491,7 +492,7 @@
:db_id database-id
:id [:in table-ids]
:active true
- :visibility_type nil))
+ :visibility_type nil {:from [(warehouse-schema-overlay/table-query)]}))
(defn- current-user-visible-table-clause
"Honey SQL `{:where …}` (plus `:with` when the filter needs a CTE) restricting Tables to those visible to the
@@ -517,7 +518,7 @@
:id [:in table-ids]
:active true
:visibility_type nil
- (current-user-visible-table-clause)))
+ (assoc (current-user-visible-table-clause) :from [(warehouse-schema-overlay/table-query)])))
(def ^:private max-visible-tables-to-consider
"Cap on the number of visible Tables fetched for fuzzy table-name matching."
@@ -532,7 +533,9 @@
:db_id database-id
:active true
:visibility_type nil
- (cond-> (assoc (current-user-visible-table-clause) :limit max-visible-tables-to-consider)
+ (cond-> (assoc (current-user-visible-table-clause)
+ :from [(warehouse-schema-overlay/table-query)]
+ :limit max-visible-tables-to-consider)
(seq excluded-table-ids)
(update :where (fn [where-clause]
(if where-clause
@@ -549,7 +552,10 @@
:db_id database-id
:active true
:visibility_type nil
- (assoc (current-user-visible-table-clause) :order-by [[:view_count :desc]] :limit limit)))
+ (assoc (current-user-visible-table-clause)
+ :from [(warehouse-schema-overlay/table-query)]
+ :order-by [[:view_count :desc]]
+ :limit limit)))
(mu/defn table-names
"Up to `limit` IDs, names, and schemas of the active, unhidden Tables in the Database with `database-id`."
@@ -559,7 +565,8 @@
:db_id database-id
:active true
:visibility_type nil
- {:limit limit}))
+ {:from [(warehouse-schema-overlay/table-query)]
+ :limit limit}))
(mu/defn active-tables-for-database
"The presentable columns of the active Tables in the Database with `database-id`, ordered by schema and name."
@@ -567,7 +574,8 @@
(t2/select [:model/Table :id :name :display_name :schema :db_id :description]
:db_id database-id
:active true
- {:order-by [[:%lower.schema :asc] [:%lower.name :asc]]}))
+ {:from [(warehouse-schema-overlay/table-query)]
+ :order-by [[:%lower.schema :asc] [:%lower.name :asc]]}))
(mu/defn active-tables-in-schema
"The presentable columns of the active Tables in `schema` of the Database with `database-id`, ordered by name."
@@ -577,13 +585,14 @@
:db_id database-id
:schema schema
:active true
- {:order-by [[:%lower.name :asc]]}))
+ {:from [(warehouse-schema-overlay/table-query)]
+ :order-by [[:%lower.name :asc]]}))
(mu/defn active-schemas-for-database
"The distinct `:schema` rows of the active Tables in the Database with `database-id`, ordered by schema."
[database-id :- ::lib.schema.id/database]
(t2/query {:select-distinct [:schema]
- :from [:metabase_table]
+ :from [(warehouse-schema-overlay/table-query {:user-settings? false})]
:where [:and [:= :db_id database-id] [:= :active true]]
:order-by [[:schema :asc]]}))
@@ -627,7 +636,7 @@
table :- :string]
(t2/select-one :model/QueryTable
{:select [[:t.id :table-id] [:t.name :table] [:t.schema :schema]]
- :from [[(t2/table-name :model/Table) :t]]
+ :from [(warehouse-schema-overlay/table-query {:alias :t, :user-settings? false})]
:where [:and
[:= :t.db_id db-id]
(table-match-clause {:schema schema :table table})]}))
@@ -642,7 +651,7 @@
[:table :string]]]]
(t2/select :model/QueryTable
{:select [[:t.id :table-id] [:t.name :table] [:t.schema :schema]]
- :from [[(t2/table-name :model/Table) :t]]
+ :from [(warehouse-schema-overlay/table-query {:alias :t, :user-settings? false})]
:where [:and
[:= :t.db_id db-id]
(into [:or] (map table-match-clause) tables)]}))
@@ -652,17 +661,17 @@
(mu/defn field
"The Field with `field-id`, or nil."
[field-id :- ::lib.schema.id/field]
- (t2/select-one :model/Field :id field-id))
+ (t2/select-one :model/Field :id field-id {:from [(warehouse-schema-overlay/field-query)]}))
(mu/defn field-fingerprint
"The fingerprint of the Field with `field-id`."
[field-id :- ::lib.schema.id/field]
- (t2/select-one-fn :fingerprint :model/Field :id field-id))
+ (t2/select-one-fn :fingerprint :model/Field :id field-id {:from [(warehouse-schema-overlay/field-query {:user-settings? false})]}))
(mu/defn field-table-ids
"A map of Field ID to Table ID for `field-ids`."
[field-ids :- [:set ::lib.schema.id/field]]
- (t2/select-fn->fn :id :table_id [:model/Field :id :table_id] :id [:in field-ids]))
+ (t2/select-fn->fn :id :table_id [:model/Field :id :table_id] :id [:in field-ids] {:from [(warehouse-schema-overlay/field-query {:user-settings? false})]}))
;;; -------------------------------------------------- Cards --------------------------------------------------
diff --git a/src/metabase/metrics/db.clj b/src/metabase/metrics/db.clj
index 02631488e6f..2b383b46d9f 100644
--- a/src/metabase/metrics/db.clj
+++ b/src/metabase/metrics/db.clj
@@ -6,6 +6,7 @@
[metabase.lib.schema.id :as lib.schema.id]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
(defn- visible-metric-cards-where
@@ -55,9 +56,9 @@
"The id and `columns` of the Fields with `field-ids`."
[columns :- [:sequential :keyword]
field-ids :- [:set ::lib.schema.id/field]]
- (t2/select (into [:model/Field :id] columns) :id [:in field-ids]))
+ (t2/select (into [:model/Field :id] columns) :id [:in field-ids] {:from [(warehouse-schema-overlay/field-query)]}))
(mu/defn field
"The Field with `field-id`, or nil."
[field-id :- ::lib.schema.id/field]
- (t2/select-one :model/Field :id field-id))
+ (t2/select-one :model/Field :id field-id {:from [(warehouse-schema-overlay/field-query)]}))
diff --git a/src/metabase/models/db.clj b/src/metabase/models/db.clj
index 81dcb75a171..dd3f556a189 100644
--- a/src/metabase/models/db.clj
+++ b/src/metabase/models/db.clj
@@ -6,6 +6,7 @@
[metabase.lib.schema.id :as lib.schema.id]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]
[toucan2.tools.identity-query :as t2.identity-query]))
@@ -82,16 +83,6 @@
[:in filter-column filter-ids])]}
(seq order-columns) (assoc :order-by (mapv (fn [column] [column :asc]) order-columns)))))
-(mu/defn table-names-reducible
- "A reducible of the id, name, and display name of every Table."
- []
- (t2/reducible-select [:model/Table :id :name :display_name]))
-
-(mu/defn field-names-reducible
- "A reducible of the id, name, and display name of every Field."
- []
- (t2/reducible-select [:model/Field :id :name :display_name]))
-
(mu/defn update-entity!
"Apply `changes` to the `model` row with `id`, returning the number updated.
@@ -101,18 +92,6 @@
changes :- [:map-of :keyword [:maybe :some]]]
(t2/update! model id changes))
-(mu/defn set-table-display-name!
- "Set the display name of the Table with `id`, returning the number updated."
- [id :- ::lib.schema.id/table
- display-name :- :string]
- (t2/update! :model/Table id {:display_name display-name}))
-
-(mu/defn set-field-display-name!
- "Set the display name of the Field with `id`, returning the number updated."
- [id :- ::lib.schema.id/field
- display-name :- :string]
- (t2/update! :model/Field id {:display_name display-name}))
-
(mu/defn insert-entity!
"Insert the `model` `row` and return the inserted instance.
@@ -184,10 +163,10 @@
[field-id :- ::lib.schema.id/field]
(t2/select :model/Field
{:with-recursive [[[:parents ^:allow-subquery {:columns [:id :name :parent_id :table_id]}]
- ^:allow-subquery {:union-all [^:allow-subquery {:from [[:metabase_field :mf]]
+ ^:allow-subquery {:union-all [^:allow-subquery {:from [(warehouse-schema-overlay/field-query {:alias :mf, :user-settings? false})]
:select [:mf.id :mf.name :mf.parent_id :mf.table_id]
:where [:= :id field-id]}
- ^:allow-subquery {:from [[:metabase_field :pf]]
+ ^:allow-subquery {:from [(warehouse-schema-overlay/field-query {:alias :pf, :user-settings? false})]
:select [:pf.id :pf.name :pf.parent_id :pf.table_id]
:join [[:parents :p] [:= :p.parent_id :pf.id]]}]}]]
:from [:parents]
@@ -196,7 +175,7 @@
(mu/defn table-ref-columns
"The id, Database id, name, and schema of the Table with `table-id`, or nil."
[table-id :- ::lib.schema.id/table]
- (t2/select-one [:model/Table :id :db_id :name :schema] :id table-id))
+ (t2/select-one [:model/Table :id :db_id :name :schema] :id table-id {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))
(mu/defn database-name
"The name of the Database with `database-id`, or nil."
@@ -218,7 +197,7 @@
[table-name :- :string
schema :- [:maybe :string]
database-id :- ::lib.schema.id/database]
- (t2/select-one-fn :id :model/Table :name table-name :schema schema :db_id database-id))
+ (t2/select-one-fn :id :model/Table :name table-name :schema schema :db_id database-id {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))
(mu/defn insert-inactive-table!
"Insert an inactive Table named `table-name` in `schema` of the Database with `database-id` and return its id."
@@ -235,7 +214,7 @@
[table-id :- ::lib.schema.id/table
field-name :- :string
parent-id :- [:maybe ms/PositiveInt]]
- (t2/select-one-pk :model/Field :table_id table-id :name field-name :parent_id parent-id))
+ (t2/select-one-pk :model/Field :table_id table-id :name field-name :parent_id parent-id {:from [(warehouse-schema-overlay/field-query {:user-settings? false})]}))
(defn- field-in-path-query
[table-id [field & rest]]
@@ -253,7 +232,7 @@
[table-id :- ::lib.schema.id/table
field-names :- [:sequential :string]]
(when (seq field-names)
- (t2/select-one-pk :model/Field (field-in-path-query table-id field-names))))
+ (t2/select-one-pk :model/Field (assoc (field-in-path-query table-id field-names) :from [(warehouse-schema-overlay/field-query {:user-settings? false})]))))
(mu/defn field-in-path
"The Field named by the last of `field-names` (each nested inside the previous, bottom-most first) under
@@ -261,7 +240,7 @@
[table-id :- [:maybe ::lib.schema.id/table]
field-names :- [:sequential :string]]
(when (seq field-names)
- (t2/select-one :model/Field (field-in-path-query table-id field-names))))
+ (t2/select-one :model/Field (assoc (field-in-path-query table-id field-names) :from [(warehouse-schema-overlay/field-query)]))))
(mu/defn insert-inactive-field!
"Insert an inactive, untyped Field named `field-name` under `parent-id` in the Table with `table-id` and return its
diff --git a/src/metabase/models/humanization.clj b/src/metabase/models/humanization.clj
deleted file mode 100644
index 158bf5707c3..00000000000
--- a/src/metabase/models/humanization.clj
+++ /dev/null
@@ -1,96 +0,0 @@
-(ns metabase.models.humanization
- "Logic related to humanization of table names and other identifiers, e.g. taking an identifier like `my_table` and
- returning a human-friendly one like `My Table`.
-
- There are currently two implementations of humanization logic, previously three.
- Which implementation is used is determined by the Setting `humanization-strategy`.
- `:simple`, which merely replaces underscores and dashes with spaces, and `:none`,
- which predictably is merely an identity function that does nothing to the results.
-
- There used to also be `:advanced`, which was the default until enough customers
- complained that we first fixed it and then the fix wasn't good enough so we removed it."
- (:require
- [metabase.models.db :as models.db]
- [metabase.settings.core :as setting :refer [defsetting]]
- [metabase.util :as u]
- [metabase.util.humanization :as u.humanization]
- [metabase.util.i18n :refer [deferred-tru tru]]
- [metabase.util.log :as log]
- [metabase.util.malli :as mu]))
-
-(declare humanization-strategy)
-
-(defn name->human-readable-name
- "Convert a name, such as `num_toucans`, to a human-readable name, such as `Num Toucans`. With one arg, this uses the
- strategy defined by the Setting `humanization-strategy`. With two args, you may specify a custom strategy (intended
- mainly for the internal implementation):
-
- (humanization-strategy! :simple)
- (name->human-readable-name \"cool_toucans\") ;-> \"Cool Toucans\"
- ;; this is the same as:
- (name->human-readable-name (humanization-strategy) \"cool_toucans\") ;-> \"Cool Toucans\"
- ;; specify a different strategy:
- (name->human-readable-name :none \"cool_toucans\") ;-> \"cool_toucans\""
- ([s]
- (name->human-readable-name (humanization-strategy) s))
- ([strategy s]
- (u.humanization/name->human-readable-name strategy s)))
-
-(defn- re-humanize-names!
- "Update all non-custom display names of all `entities-reducible` rows (e.g. Table or Field ones), using `model` for
- logging and `set-display-name!` to apply the change."
- [old-strategy model entities-reducible set-display-name!]
- (run! (fn [{id :id, internal-name :name, display-name :display_name}]
- (let [old-strategy-display-name (name->human-readable-name old-strategy internal-name)
- new-strategy-display-name (name->human-readable-name internal-name)
- custom-display-name? (not= old-strategy-display-name display-name)]
- (when (and (not= display-name new-strategy-display-name)
- (not custom-display-name?))
- (log/infof "Updating display name for %s '%s': '%s' -> '%s'"
- (name model) internal-name display-name new-strategy-display-name)
- (set-display-name! id new-strategy-display-name))))
- (entities-reducible)))
-
-(mu/defn- re-humanize-table-and-field-names!
- "Update the non-custom display names of all Tables & Fields in the database using new values obtained from
- the (obstensibly swapped implementation of) `name->human-readable-name`."
- [old-strategy :- :keyword]
- (re-humanize-names! old-strategy :model/Table models.db/table-names-reducible models.db/set-table-display-name!)
- (re-humanize-names! old-strategy :model/Field models.db/field-names-reducible models.db/set-field-display-name!))
-
-(defn- set-humanization-strategy! [new-value]
- (let [new-strategy (keyword (or new-value :simple))]
- ;; check to make sure `new-strategy` is a valid strategy, or throw an Exception it is it not.
- (when-not (get-method u.humanization/name->human-readable-name new-strategy)
- (throw (IllegalArgumentException.
- (tru "Invalid humanization strategy ''{0}''. Valid strategies are: {1}"
- new-strategy (keys (methods u.humanization/name->human-readable-name))))))
- (let [old-strategy (setting/get-value-of-type :keyword :humanization-strategy)]
- ;; ok, now set the new value
- (setting/set-value-of-type! :keyword :humanization-strategy new-value)
- ;; now rehumanize all the Tables and Fields using the new strategy.
- ;; TODO: Should we do this in a background thread because it is potentially slow?
- ;; https://github.com/metabase/metabase/issues/39406
- (log/infof "Changing Table & Field names humanization strategy from '%s' to '%s'"
- (name old-strategy) (name new-strategy))
- (re-humanize-table-and-field-names! old-strategy))))
-
-;;; these kondo warnings are ignored for now because I'm planning on moving this namespace out of `util` to eliminate
-;;; the dependency of `util` of `settings` -- will fix them after this namespace gets moved. -- Cam
-#_{:clj-kondo/ignore [:metabase/defsetting-namespace]}
-(defsetting ^{:added "0.28.0"} humanization-strategy
- (deferred-tru
- (str "To make table and field names more human-friendly, Metabase will replace dashes and underscores in them "
- "with spaces. We’ll capitalize each word while at it, so ‘last_visited_at’ will become ‘Last Visited At’."))
- :type :keyword
- :default :simple
- :visibility :settings-manager
- :export? true
- :audit :raw-value
- :getter (fn []
- (let [strategy (setting/get-value-of-type :keyword :humanization-strategy)
- valid-values (set (keys (methods u.humanization/name->human-readable-name)))
- valid-strategy? (contains? valid-values strategy)]
- (when (not valid-strategy?) (log/warn (u/format-color :yellow "Invalid humanization strategy '%s'. Defaulting to 'simple'" strategy)))
- (if valid-strategy? strategy :simple)))
- :setter set-humanization-strategy!)
diff --git a/src/metabase/models/resolution.clj b/src/metabase/models/resolution.clj
index d3785288fdd..b914987f5ed 100644
--- a/src/metabase/models/resolution.clj
+++ b/src/metabase/models/resolution.clj
@@ -60,6 +60,7 @@
:model/ExplorationThreadTimeline metabase.explorations.models.exploration-thread-timeline
:model/Field metabase.warehouse-schema.models.field
:model/FieldUserSettings metabase.warehouse-schema.models.field-user-settings
+ :model/TableUserSettings metabase.warehouse-schema.models.table-user-settings
:model/FieldValues metabase.warehouse-schema.models.field-values
:model/Glossary metabase.glossary.models.glossary
:model/Sandbox metabase-enterprise.sandbox.models.sandbox
diff --git a/src/metabase/models/serialization.clj b/src/metabase/models/serialization.clj
index 85aa3371e3b..c1353901a60 100644
--- a/src/metabase/models/serialization.clj
+++ b/src/metabase/models/serialization.clj
@@ -386,6 +386,11 @@
(assert (contains? m k2)
(format "Transform must define one of %s or %s" k1 k2))))
+(defn primary-key
+ "The primary key column of `model-name`'s model; serialization keys every model by one column."
+ [model-name]
+ (first (t2/primary-keys (keyword "model" model-name))))
+
(defn extract-one
"Extracts a single entity retrieved from the database into a portable map with `:serdes/meta` attached.
`(extract-one \"ModelName\" opts entity)`
@@ -451,24 +456,28 @@
(defn- transform->nested [transform opts batch]
(let [backward-fk (:backward-fk transform)
+ pk (primary-key (name (t2/model (first batch))))
entities (-> (extract-query (name (:model transform))
(assoc opts
:filter-column backward-fk
- :filter-ids (mapv :id batch)
+ :filter-ids (mapv pk batch)
::nested-fetch true))
t2.realize/realize)]
(group-by backward-fk entities)))
(defn- extract-batch-nested [model-name opts batch]
- (let [spec (*make-spec* model-name opts)]
+ (let [spec (*make-spec* model-name opts)
+ pk (primary-key model-name)]
(reduce-kv (fn [batch k transform]
(if-not (::nested transform)
batch
- (mi/instances-with-hydrated-data batch k #(transform->nested transform opts batch) :id)))
+ (mi/instances-with-hydrated-data batch k #(transform->nested transform opts batch) pk)))
batch
(:transform spec))))
-(defn- extract-reducible-nested [model-name opts reducible]
+(defn extract-reducible-nested
+ "Hydrate the nested transforms of `model-name`'s spec onto the entities of `reducible`."
+ [model-name opts reducible]
(eduction (comp (map t2.realize/realize)
(partition-all (or (:batch-limit opts)
extract-nested-batch-limit))
@@ -1876,10 +1885,13 @@
:export #(*export-fk* % model)
:import #(*import-fk* % model)))))
-(defn nested "Nested entities" [model backward-fk opts]
- (let [model-name (name model)
- sorter (:sort-by opts :created_at)
- key-field (:key-field opts :entity_id)]
+(defn nested
+ "Nested entities; `opts` may give `:sort-by`, `:key-field` and `:delete-children!`, a fn of the parent id."
+ [model backward-fk opts]
+ (let [model-name (name model)
+ sorter (:sort-by opts :created_at)
+ key-field (:key-field opts :entity_id)
+ delete-children! (:delete-children! opts #(models.db/delete-children! model backward-fk %))]
{::nested true
:model model
:backward-fk backward-fk
@@ -1895,10 +1907,10 @@
(catch Exception e
(throw (ex-info (format "Error extracting nested %s" model)
{:model model
- :parent-id (:id current)}
+ :parent-id (some->> (t2/model current) name primary-key (get current))}
e)))))
:import-with-context (fn [current _ lst]
- (let [parent-id (:id current)
+ (let [parent-id (get current (primary-key (name (t2/model current))))
first-eid (some->> (first lst)
(entity-id model-name))
enrich (fn [ingested]
@@ -1907,7 +1919,7 @@
(update :serdes/meta #(or % [{:model model-name :id (get ingested key-field)}]))))]
(cond
(nil? first-eid) ; no entity id, just drop existing stuff
- (do (models.db/delete-children! model backward-fk parent-id)
+ (do (delete-children! parent-id)
(doseq [ingested lst]
(load-one! (enrich ingested) nil)))
diff --git a/src/metabase/parameters/db.clj b/src/metabase/parameters/db.clj
index 7468c6aed60..8f919a6c9e8 100644
--- a/src/metabase/parameters/db.clj
+++ b/src/metabase/parameters/db.clj
@@ -8,6 +8,7 @@
[metabase.lib.schema.id :as lib.schema.id]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
(defn- format-union
@@ -24,9 +25,9 @@
[field-id mapping-type]
^:allow-subquery
{:select [[:dest.id :id] [^:allow-raw-sql [:inline mapping-type] :mapping_type]]
- :from [[:metabase_field :source]]
+ :from [(warehouse-schema-overlay/field-query {:alias :source})]
:left-join [[:metabase_table :table] [:= :source.table_id :table.id]
- [:metabase_field :dest] [:= :dest.table_id :table.id]]
+ (warehouse-schema-overlay/field-query {:alias :dest}) [:= :dest.table_id :table.id]]
:where [:and
[:= :source.id field-id]
(mdb/isa :source.semantic_type :type/PK)
@@ -54,7 +55,7 @@
(implicit-pk->name-mapping-query
^:allow-subquery
{:select [:fk_target_field_id]
- :from [:metabase_field]
+ :from [(warehouse-schema-overlay/field-query)]
:where [:and
[:= :id field-id]
(mdb/isa :semantic_type :type/FK)]
@@ -68,32 +69,32 @@
(mu/defn field
"The Field with `field-id`, or nil."
[field-id :- ::lib.schema.id/field]
- (t2/select-one :model/Field :id field-id))
+ (t2/select-one :model/Field :id field-id {:from [(warehouse-schema-overlay/field-query)]}))
(mu/defn fields
"The Fields with `field-ids`."
[field-ids :- [:set ::lib.schema.id/field]]
- (t2/select :model/Field :id [:in field-ids]))
+ (t2/select :model/Field :id [:in field-ids] {:from [(warehouse-schema-overlay/field-query)]}))
(mu/defn fields-fk-info
"The id, FK target, and semantic type of the Fields with `field-ids`."
[field-ids :- [:set ::lib.schema.id/field]]
- (t2/select [:model/Field :id :fk_target_field_id :semantic_type] :id [:in field-ids]))
+ (t2/select [:model/Field :id :fk_target_field_id :semantic_type] :id [:in field-ids] {:from [(warehouse-schema-overlay/field-query)]}))
(mu/defn field-fk-target-field-id
"The FK target Field id of the Field with `field-id`, or nil."
[field-id :- ::lib.schema.id/field]
- (t2/select-one-fn :fk_target_field_id :model/Field field-id))
+ (t2/select-one-fn :fk_target_field_id :model/Field :id field-id {:from [(warehouse-schema-overlay/field-query)]}))
(mu/defn field-base-type
"The base type of the Field with `field-id`, or nil."
[field-id :- ::lib.schema.id/field]
- (t2/select-one-fn :base_type :model/Field :id field-id))
+ (t2/select-one-fn :base_type :model/Field :id field-id {:from [(warehouse-schema-overlay/field-query {:user-settings? false})]}))
(mu/defn field-name
"The name of the Field with `field-id`, or nil."
[field-id :- ::lib.schema.id/field]
- (t2/select-one-fn :name :model/Field :id field-id))
+ (t2/select-one-fn :name :model/Field :id field-id {:from [(warehouse-schema-overlay/field-query {:user-settings? false})]}))
(mu/defn full-field-values-exist?
"Whether complete, non-remapped FieldValues of type `full` exist for the Field with `field-id`."
@@ -141,13 +142,14 @@
(t2/select (into [:model/Field] columns)
:table_id [:in table-ids]
:semantic_type (mdb/isa :type/Name)
- :active true))
+ :active true
+ {:from [(warehouse-schema-overlay/field-query)]}))
(mu/defn fields-with-columns
"The `columns` of the Fields with `field-ids`."
[columns :- [:sequential :keyword]
field-ids :- [:set ::lib.schema.id/field]]
- (t2/select (into [:model/Field] columns) :id [:in field-ids]))
+ (t2/select (into [:model/Field] columns) :id [:in field-ids] {:from [(warehouse-schema-overlay/field-query)]}))
(mu/defn fk-relationships-for-database
"Rows describing FK -> PK Field relationships (`:f1`/`:t1` FK Field/Table ids, `:f2`/`:t2` PK Field/Table ids)
@@ -157,12 +159,12 @@
[:fk-table.id :t1]
[:pk-field.id :f2]
[:pk-field.table_id :t2]]
- :from [[:metabase_field :fk-field]]
+ :from [(warehouse-schema-overlay/field-query {:alias :fk-field})]
:left-join [[:metabase_table :fk-table] [:and [:= :fk-field.table_id :fk-table.id]
:fk-table.active]
[:metabase_database :database] [:= :fk-table.db_id :database.id]
- [:metabase_field :pk-field] [:and [:= :fk-field.fk_target_field_id :pk-field.id]
- :pk-field.active]]
+ (warehouse-schema-overlay/field-query {:alias :pk-field}) [:and [:= :fk-field.fk_target_field_id :pk-field.id]
+ :pk-field.active]]
:where [:and
[:= :database.id database-id]
[:not= :fk-field.fk_target_field_id nil]
diff --git a/src/metabase/permissions/db.clj b/src/metabase/permissions/db.clj
index 3f40df968c1..9d1bb7fa648 100644
--- a/src/metabase/permissions/db.clj
+++ b/src/metabase/permissions/db.clj
@@ -14,6 +14,7 @@
[metabase.util.honey-sql-2 :as h2x]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
;;; --------------------------------------------- DataPermissions ---------------------------------------------
@@ -748,36 +749,37 @@
(mu/defn table-location
"The ID, Database ID, and schema of the Table with `table-id`."
[table-id :- ::lib.schema.id/table]
- (t2/select-one [:model/Table :id :db_id :schema] :id table-id))
+ (t2/select-one [:model/Table :id :db_id :schema] :id table-id {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))
(mu/defn table-database-id
"The Database ID of the Table with `table-id`."
[table-id :- ::lib.schema.id/table]
- (t2/select-one-fn :db_id :model/Table table-id))
+ (t2/select-one-fn :db_id :model/Table :id table-id {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))
(mu/defn table-database-ids
"The ID and Database ID of the Tables with `table-ids`."
[table-ids :- [:set ::lib.schema.id/table]]
- (t2/select [:model/Table :id :db_id] :id [:in table-ids]))
+ (t2/select [:model/Table :id :db_id] :id [:in table-ids] {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))
(mu/defn active-table-locations-for-database
"The ID, Database ID, and schema of the active Tables of the Database with `database-id`."
[database-id :- ::lib.schema.id/database]
- (t2/select [:model/Table :id :db_id :schema] :db_id database-id :active true))
+ (t2/select [:model/Table :id :db_id :schema] :db_id database-id :active true {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))
(mu/defn table-ids-and-schemas-excluding
"The ID and schema of the Tables of the Database with `database-id` other than `excluded-table-ids`."
[database-id :- ::lib.schema.id/database
excluded-table-ids :- [:sequential ::lib.schema.id/table]]
(t2/select [:model/Table :id :schema]
- {:where [:and
+ {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]
+ :where [:and
[:= :db_id database-id]
[:not [:in :id excluded-table-ids]]]}))
(mu/defn field-visibility-info
"The ID, visibility type, and Table ID of the Fields with `field-ids`."
[field-ids :- [:set ::lib.schema.id/field]]
- (t2/select [:model/Field :id :visibility_type :table_id] :id [:in field-ids]))
+ (t2/select [:model/Field :id :visibility_type :table_id] :id [:in field-ids] {:from [(warehouse-schema-overlay/field-query)]}))
(mu/defn instance-by-id
"The instance of `model` with `id`, or nil."
diff --git a/src/metabase/permissions/models/data_permissions/sql.clj b/src/metabase/permissions/models/data_permissions/sql.clj
index 6cb3f7e865a..93c339c1ae7 100644
--- a/src/metabase/permissions/models/data_permissions/sql.clj
+++ b/src/metabase/permissions/models/data_permissions/sql.clj
@@ -4,7 +4,8 @@
[metabase.permissions.published-tables :as published-tables]
[metabase.permissions.schema :as permissions.schema]
[metabase.util.honey-sql-2 :as h2x]
- [metabase.util.malli :as mu])
+ [metabase.util.malli :as mu]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay])
(:import
(clojure.lang PersistentVector)))
@@ -121,6 +122,12 @@
::permissions.schema/data-permission-type
[:or ::permissions.schema/data-permission-value [:tuple ::permissions.schema/data-permission-value [:enum :most :least]]]])
+(defn- table-source
+ "The plain `metabase_table`, aliased `mt`: permission queries read only sync-owned columns, and the overlay's
+ derived table is costly here and breaks on MariaDB."
+ []
+ (warehouse-schema-overlay/table-query {:alias :mt, :user-settings? false}))
+
(mu/defn visible-table-filter-select
"Selects a column from tables that are visible to the provided user given a mapping of permission types to the required value or the required
value and a directive if we should test against the most or least permissive permission the user has.
@@ -135,7 +142,7 @@
{:select [(case select-column
:id :mt.id
:db_id :mt.db_id)]
- :from [[:metabase_table :mt]]
+ :from [(table-source)]
:where (if (or is-superuser?
(and is-data-analyst?
(contains? permission-mapping :perms/manage-table-metadata)))
@@ -227,7 +234,7 @@
^:allow-subquery
{:select [:mt.id :dp.perm_type :dp.perm_value]
:from [[:data_permissions :dp]]
- :join [[:metabase_table :mt] [:= :mt.id :dp.table_id]]
+ :join [(table-source) [:= :mt.id :dp.table_id]]
:where (into [:and
[:not= :dp.table_id nil]
user-groups-clause
@@ -237,7 +244,7 @@
^:allow-subquery
{:select [:mt.id :dp.perm_type :dp.perm_value]
:from [[:data_permissions :dp]]
- :join [[:metabase_table :mt] [:= :mt.db_id :dp.db_id]]
+ :join [(table-source) [:= :mt.db_id :dp.db_id]]
:where (into [:and
[:= :dp.table_id nil]
user-groups-clause
@@ -262,7 +269,7 @@
permission-mapping :- PermissionMapping]
^:allow-subquery
{:select [:mt.id :dp.group_id :dp.perm_type :dp.perm_value]
- :from [[:metabase_table :mt]]
+ :from [(table-source)]
:join [[:data_permissions :dp] [:or
[:and
[:= :mt.db_id :dp.db_id]
diff --git a/src/metabase/permissions_rest/db.clj b/src/metabase/permissions_rest/db.clj
index 731dc3ed262..ea10fa89ed5 100644
--- a/src/metabase/permissions_rest/db.clj
+++ b/src/metabase/permissions_rest/db.clj
@@ -6,6 +6,7 @@
[metabase.util :as u]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
(defn- managed-groups-clause
@@ -168,4 +169,5 @@
(mu/defn tables-for-databases
"The id, Database id, and schema of the Tables of the Databases with `database-ids`."
[database-ids :- [:set ::lib.schema.id/database]]
- (t2/select [:model/Table :id :db_id :schema] :db_id [:in database-ids]))
+ (t2/select [:model/Table :id :db_id :schema] :db_id [:in database-ids]
+ {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))
diff --git a/src/metabase/queries/db.clj b/src/metabase/queries/db.clj
index 5f3b8f9e9ee..8c6079ae60b 100644
--- a/src/metabase/queries/db.clj
+++ b/src/metabase/queries/db.clj
@@ -10,6 +10,7 @@
[metabase.util.honey-sql-2 :as h2x]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
;;; ------------------------------------------------ Cards ------------------------------------------------
@@ -221,14 +222,14 @@
[:table.name :table-name]
[:table.db_id :field-db-id]]
:from [[:metabase_field :field]]
- :left-join [[:metabase_table :table]
+ :left-join [(warehouse-schema-overlay/table-query {:alias :table, :user-settings? false})
[:= :field.table_id :table.id]]
:where [:in :field.id field-ids]}))
(mu/defn field-table-ids
"The set of Table IDs of the Fields with `field-ids`."
[field-ids :- [:set ::lib.schema.id/field]]
- (t2/select-fn-set :table_id :model/Field :id [:in field-ids]))
+ (t2/select-fn-set :table_id :model/Field :id [:in field-ids] {:from [(warehouse-schema-overlay/field-query {:user-settings? false})]}))
(mu/defn snippets
"The NativeQuerySnippets with `snippet-ids`."
diff --git a/src/metabase/queries_rest/db.clj b/src/metabase/queries_rest/db.clj
index c9923f480c4..d388a87f650 100644
--- a/src/metabase/queries_rest/db.clj
+++ b/src/metabase/queries_rest/db.clj
@@ -8,6 +8,7 @@
[metabase.queries.schema :as queries.schema]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
(def ^:private order-by-name {:order-by [[:%lower.name :asc]]})
@@ -75,14 +76,14 @@
"The Database id of the Table of the Segment with `segment-id`, or nil."
[segment-id :- ::lib.schema.id/segment]
(t2/select-one-fn :db_id :model/Table {:select [:t.db_id]
- :from [[:metabase_table :t]]
+ :from [(warehouse-schema-overlay/table-query {:alias :t, :user-settings? false})]
:join [[:segment :m] [:= :t.id :m.table_id]]
:where [:= :m.id segment-id]}))
(mu/defn table-database-id
"The Database id of the Table with `table-id`, or nil."
[table-id :- ::lib.schema.id/table]
- (t2/select-one-fn :db_id :model/Table, :id table-id))
+ (t2/select-one-fn :db_id :model/Table, :id table-id {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))
(mu/defn card
"The Card with `card-id`, or nil."
diff --git a/src/metabase/query_permissions/db.clj b/src/metabase/query_permissions/db.clj
index 68da545024e..91a00465d70 100644
--- a/src/metabase/query_permissions/db.clj
+++ b/src/metabase/query_permissions/db.clj
@@ -4,6 +4,7 @@
(:require
[metabase.lib.schema.id :as lib.schema.id]
[metabase.util.malli :as mu]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
(mu/defn card-collection-id
@@ -20,9 +21,10 @@
(mu/defn field-table-ids
"The set of Table ids of the Fields with `field-ids`."
[field-ids :- [:or [:set ::lib.schema.id/field] [:sequential ::lib.schema.id/field]]]
- (t2/select-fn-set :table_id :model/Field :id [:in field-ids]))
+ (t2/select-fn-set :table_id :model/Field :id [:in field-ids] {:from [(warehouse-schema-overlay/field-query {:user-settings? false})]}))
(mu/defn table-id->database-id
"A map of Table id to Database id for the Tables with `table-ids`."
[table-ids :- [:set ::lib.schema.id/table]]
- (t2/select-pk->fn :db_id :model/Table :id [:in table-ids]))
+ (t2/select-pk->fn :db_id :model/Table :id [:in table-ids]
+ {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))
diff --git a/src/metabase/query_processor/db.clj b/src/metabase/query_processor/db.clj
index f68fdabbc1b..05f2e61aa80 100644
--- a/src/metabase/query_processor/db.clj
+++ b/src/metabase/query_processor/db.clj
@@ -7,6 +7,7 @@
[metabase.lib.schema.id :as lib.schema.id]
[metabase.queries.schema :as queries.schema]
[metabase.util.malli :as mu]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
^{:clj-kondo/ignore [:discouraged-namespace]}
[toucan2.core :as t2]))
@@ -17,7 +18,7 @@
(mu/defn table
"The Table with `table-id`, or nil."
[table-id :- ::lib.schema.id/table]
- (t2/select-one :model/Table :id table-id))
+ (t2/select-one :model/Table :id table-id {:from [(warehouse-schema-overlay/table-query)]}))
(mu/defn source-card-metadata
"The entity id, result metadata, and type of the Card with `card-id`, or nil."
diff --git a/src/metabase/search/in_place/legacy.clj b/src/metabase/search/in_place/legacy.clj
index 37bfaff577e..817e2749e80 100644
--- a/src/metabase/search/in_place/legacy.clj
+++ b/src/metabase/search/in_place/legacy.clj
@@ -19,6 +19,7 @@
[metabase.util.honey-sql-2 :as h2x]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
(def ^:private HoneySQLColumn
@@ -659,12 +660,12 @@
(defmethod search-query-for-model "measure"
[model search-ctx]
(-> (base-query-for-model model search-ctx)
- (sql.helpers/left-join [:metabase_table :table] [:= :measure.table_id :table.id])))
+ (sql.helpers/left-join (warehouse-schema-overlay/table-query {:alias :table}) [:= :measure.table_id :table.id])))
(defmethod search-query-for-model "segment"
[model search-ctx]
(-> (base-query-for-model model search-ctx)
- (sql.helpers/left-join [:metabase_table :table] [:= :segment.table_id :table.id])))
+ (sql.helpers/left-join (warehouse-schema-overlay/table-query {:alias :table}) [:= :segment.table_id :table.id])))
(defmethod search-query-for-model "table"
[model {:keys [current-user-perms table-db-id], :as search-ctx}]
diff --git a/src/metabase/search/ingestion/query.clj b/src/metabase/search/ingestion/query.clj
index 0cecddf214a..dfa593a381c 100644
--- a/src/metabase/search/ingestion/query.clj
+++ b/src/metabase/search/ingestion/query.clj
@@ -54,7 +54,9 @@
(remove (comp search-terms key)))))
[:attrs :render-terms])
fn-selects))
- :from [[(t2/table-name (:model spec)) :this]]
+ :from [(if-let [source (:source spec)]
+ (source)
+ [(t2/table-name (:model spec)) :this])]
:where (:where spec [:= [:inline 1] [:inline 1]])
:left-join (when (:joins spec)
(into []
diff --git a/src/metabase/search/spec.clj b/src/metabase/search/spec.clj
index 173b1cf0fb6..c92c5ce328f 100644
--- a/src/metabase/search/spec.clj
+++ b/src/metabase/search/spec.clj
@@ -199,6 +199,7 @@
[:map-of :keyword [:or fn? true?]]]]
[:embedding-exclude {:optional true} [:set :keyword]]
[:render-terms [:map-of NonAttrKey AttrValue]]
+ [:source {:optional true} fn?]
[:where {:optional true} vector?]
[:bookmark {:optional true} vector?]
[:joins {:optional true} JoinMap]])
@@ -399,6 +400,10 @@
Spec keys:
- `:model` - Toucan model keyword (required)
- `:attrs` - Map of search index attributes (required)
+ - `:source` - A thunk returning what `:this` reads from, as a `[source :this]` pair, when the indexable rows are
+ not simply the model's own table -- a Table, whose user-set values live in a side-car. A thunk because the spec
+ map is evaluated as the namespace loads, and resolving another model there would be a load cycle. Defaults to
+ the model's own table.
- `:search-terms` - Searchable text fields: a vector of column keywords, or a map of
column keyword to either `true` (use the raw value) or a transform fn applied for
full-text search (required)
diff --git a/src/metabase/segments/db.clj b/src/metabase/segments/db.clj
index 2c67d0d10ee..c8181f67740 100644
--- a/src/metabase/segments/db.clj
+++ b/src/metabase/segments/db.clj
@@ -6,6 +6,7 @@
[metabase.lib.schema.id :as lib.schema.id]
[metabase.segments.schema :as segments.schema]
[metabase.util.malli :as mu]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
(mu/defn insert-segment!
@@ -35,7 +36,7 @@
(mu/defn table-database-ids
"The set of Database ids of the Tables with `table-ids`."
[table-ids :- [:set ::lib.schema.id/table]]
- (t2/select-fn-set :db_id :model/Table :id [:in table-ids]))
+ (t2/select-fn-set :db_id :model/Table :id [:in table-ids] {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))
(mu/defn update-segment!
"Apply `changes` to the Segment with `id`."
@@ -46,17 +47,17 @@
(mu/defn table
"The Table with `table-id`, or nil."
[table-id :- ::lib.schema.id/table]
- (t2/select-one :model/Table :id table-id))
+ (t2/select-one :model/Table :id table-id {:from [(warehouse-schema-overlay/table-query)]}))
(mu/defn table-database-id
"The Database id of the Table with `table-id`, or nil."
[table-id :- ::lib.schema.id/table]
- (t2/select-one-fn :db_id :model/Table :id table-id))
+ (t2/select-one-fn :db_id :model/Table :id table-id {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))
(mu/defn table-perms-columns
"The Database id, schema, and id of the Table with `table-id`, or nil."
[table-id :- ::lib.schema.id/table]
- (t2/select-one [:model/Table :db_id :schema :id] :id table-id))
+ (t2/select-one [:model/Table :db_id :schema :id] :id table-id {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))
(mu/defn collections
"The Collections with `collection-ids`."
diff --git a/src/metabase/sql_tools/db.clj b/src/metabase/sql_tools/db.clj
index 67f7f759274..742aa6e8fa0 100644
--- a/src/metabase/sql_tools/db.clj
+++ b/src/metabase/sql_tools/db.clj
@@ -5,6 +5,7 @@
[metabase.lib.schema.id :as lib.schema.id]
[metabase.util :as u]
[metabase.util.malli :as mu]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
(mu/defn active-visible-table-ids-by-name
@@ -13,7 +14,8 @@
[database-id :- ::lib.schema.id/database
table-names :- [:sequential :string]]
(t2/select-pks-set :model/Table
- {:where [:and
+ {:from [(warehouse-schema-overlay/table-query)]
+ :where [:and
[:= :db_id database-id]
;; `lower()` cannot use an index on the name column, but it still beats fetching every
;; row for the Database.
diff --git a/src/metabase/sync/analyze/data_sensitivity.clj b/src/metabase/sync/analyze/data_sensitivity.clj
index da8b7de2e38..b2a8c7c19e2 100644
--- a/src/metabase/sync/analyze/data_sensitivity.clj
+++ b/src/metabase/sync/analyze/data_sensitivity.clj
@@ -3,8 +3,8 @@
[[metabase.analyze.core/infer-data-sensitivity]], writing `:PUBLIC` when no rule matches so each field is scanned
once. Selects fields whose `data_sensitivity` is `NULL` (or `PUBLIC` too under `force?`) across every active table
in the database, hidden and cruft tables included. Inert unless [[metabase.sync.settings/data-sensitivity-scan-enabled]]
- is true, except through [[scan-data-sensitivity!]]. User-set labels are protected by the `FieldUserSettings`
- overlay applied on every Field update. [[reset-data-sensitivity!]] clears the classifier's own labels so a rule
+ is true, except through [[scan-data-sensitivity!]]. User-set labels live in `FieldUserSettings`, which readers apply
+ over the classifier's label, so the classifier never overwrites them. [[reset-data-sensitivity!]] clears the classifier's own labels so a rule
change can reach fields that already carry a category."
(:require
[metabase.analyze.core :as analyze]
@@ -109,11 +109,8 @@
n))
(mu/defn scan-data-sensitivity! :- Stats
- "REPL entry point: scan a Database or a single Table regardless of the `data-sensitivity-scan-enabled` setting.
- With `:force? true` fields already labeled `:PUBLIC` are rescanned too; fields carrying a category are not. With
- `:reset? true` every classifier-written label is cleared first via [[reset-data-sensitivity!]] so the whole scope
- is recomputed under the current rules, and `:fields-reset` is added to the stats. User-set labels survive in
- every mode because the `FieldUserSettings` overlay is applied on every Field update."
+ "Scan `database-or-table` for data sensitivity regardless of the setting; `:force?` rescans `:PUBLIC` Fields and
+ `:reset?` clears the classifier's labels first."
[database-or-table :- [:or i/DatabaseInstance i/TableInstance]
& {:keys [force? reset?]} :- [:maybe [:map
[:force? {:optional true} [:maybe :boolean]]
diff --git a/src/metabase/sync/db.clj b/src/metabase/sync/db.clj
index 2d51533589a..d2e6fb1bccc 100644
--- a/src/metabase/sync/db.clj
+++ b/src/metabase/sync/db.clj
@@ -1,6 +1,10 @@
(ns metabase.sync.db
- "Application database queries for the sync module. Every function here is a direct Toucan 2 call with no additional
- logic, so the rest of the module never talks to `toucan2.core` itself."
+ "Application database queries for the sync module. Table reads here ask for the rows sync itself wrote
+ (`{:user-settings? false}`): it reconciles them against what the warehouse reports, so it has to see a Table as it
+ recorded it, whatever a reader would be shown instead.
+
+ Every function here is a direct Toucan 2 call with no additional logic, so the rest of the module never talks to
+ `toucan2.core` itself."
(:require
[clojure.set :as set]
[honey.sql :as sql]
@@ -10,6 +14,7 @@
[metabase.util.honey-sql-2 :as h2x]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[metabase.warehouse-schema.models.table :as table]
[metabase.warehouse-schema.schema :as warehouse-schema.schema]
[metabase.warehouses.schema :as warehouses.schema]
@@ -71,39 +76,39 @@
(mu/defn table
"The Table with `table-id`, or nil."
[table-id :- ::lib.schema.id/table]
- (t2/select-one :model/Table :id table-id))
+ (t2/select-one :model/Table :id table-id {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))
(mu/defn table-in-database
"The Table with `table-id` in the Database with `database-id`, or nil."
[database-id :- ::lib.schema.id/database
table-id :- ::lib.schema.id/table]
- (t2/select-one :model/Table :db_id database-id :id table-id))
+ (t2/select-one :model/Table :db_id database-id :id table-id {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))
(mu/defn table-by-name
"The Table named `table-name` in the Database with `database-id`, or nil."
[database-id :- ::lib.schema.id/database
table-name :- :string]
- (t2/select-one :model/Table :db_id database-id :name table-name))
+ (t2/select-one :model/Table :db_id database-id :name table-name {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))
(mu/defn table-by-schema-and-name
"The Table named `table-name` in `schema` of the Database with `database-id`, or nil."
[database-id :- ::lib.schema.id/database
schema :- [:maybe :string]
table-name :- :string]
- (t2/select-one :model/Table :db_id database-id :name table-name :schema schema))
+ (t2/select-one :model/Table :db_id database-id :name table-name :schema schema {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))
(mu/defn inactive-table-by-schema-and-name
"The inactive Table named `table-name` in `schema` of the Database with `database-id`, or nil."
[database-id :- ::lib.schema.id/database
schema :- [:maybe :string]
table-name :- :string]
- (t2/select-one :model/Table :db_id database-id :schema schema :name table-name :active false))
+ (t2/select-one :model/Table :db_id database-id :schema schema :name table-name :active false {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))
(mu/defn active-table-id-by-name
"The ID of the active Table named `table-name` in the Database with `database-id`, or nil."
[database-id :- ::lib.schema.id/database
table-name :- :string]
- (t2/select-one-pk :model/Table :db_id database-id :name table-name :active true))
+ (t2/select-one-pk :model/Table :db_id database-id :name table-name :active true {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))
(mu/defn sync-tables-by-lower-name-and-schema
"The synced Tables of the Database with `database-id` whose lower-cased name and schema match."
@@ -114,7 +119,8 @@
:db_id database-id
:%lower.name lower-name
:%lower.schema lower-schema
- {:where sync-tables-clause}))
+ {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]
+ :where sync-tables-clause}))
(mu/defn tables-by-name
"The `columns` of the Tables of the Database with `database-id` named one of `table-names`."
@@ -134,32 +140,33 @@
:active false
:archived_at nil
:transform_target false
- :deactivated_at [:< (h2x/add-interval-honeysql-form (app-db/db-type) :%now amount unit)]))
+ :deactivated_at [:< (h2x/add-interval-honeysql-form (app-db/db-type) :%now amount unit)] {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))
(mu/defn table-database-ids
"A map of Table ID to Database ID for `table-ids`."
[table-ids :- [:set ::lib.schema.id/table]]
- (t2/select-pk->fn :db_id :model/Table :id [:in table-ids]))
+ (t2/select-pk->fn :db_id :model/Table :id [:in table-ids] {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))
(mu/defn table-schemas-reducible
"Reducible `:schema` rows of the Tables of the Database with `database-id`."
[database-id :- ::lib.schema.id/database]
- (t2/reducible-select [:model/Table :schema] :db_id database-id))
+ (t2/reducible-select [:model/Table :schema] :db_id database-id {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))
(mu/defn active-table-ids-reducible
"Reducible `:id` rows of the active Tables of the Database with `database-id`."
[database-id :- ::lib.schema.id/database]
- (t2/reducible-select [:model/Table :id] :db_id database-id :active true))
+ (t2/reducible-select [:model/Table :id] :db_id database-id :active true {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))
(mu/defn active-table-count
"The number of active Tables in the Database with `database-id`."
[database-id :- ::lib.schema.id/database]
- (t2/count :model/Table :db_id database-id :active true))
+ (t2/count :model/Table :db_id database-id :active true {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))
(mu/defn sync-table-ids
"The IDs of the synced Tables of the Database with `database-id`."
[database-id :- ::lib.schema.id/database]
- (t2/select-fn-vec :id :model/Table :db_id database-id {:where sync-tables-clause}))
+ (t2/select-fn-vec :id :model/Table :db_id database-id {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]
+ :where sync-tables-clause}))
(mu/defn sync-table-schemas
"The distinct `:schema` rows of the synced Tables of the Database with `database-id`."
@@ -171,7 +178,8 @@
(mu/defn sync-tables-count
"The number of synced Tables in the Database with `database-id`."
[database-id :- ::lib.schema.id/database]
- (t2/count :model/Table :db_id database-id {:where sync-tables-clause}))
+ (t2/count :model/Table :db_id database-id {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]
+ :where sync-tables-clause}))
(mu/defn sync-tables-reducible
"Reducible synced Tables of the Database with `database-id` ordered by schema and name, optionally narrowed to
@@ -181,7 +189,8 @@
table-names :- [:maybe [:or [:set :string] [:sequential :string]]]]
(t2/reducible-select :model/Table
:db_id database-id
- {:where [:and sync-tables-clause
+ {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]
+ :where [:and sync-tables-clause
(when (seq schema-names) [:in :schema schema-names])
(when (seq table-names) [:in :name table-names])]
:order-by [[:schema :asc] [:name :asc]]}))
@@ -191,10 +200,10 @@
[database-id :- ::lib.schema.id/database]
(t2/reducible-select :model/Table
{:select [:t.*]
- :from [[(t2/table-name :model/Table) :t]]
+ :from [(warehouse-schema-overlay/table-query {:alias :t})]
:left-join [[^:allow-subquery {:select [:table_id
[[:min :last_analyzed] :earliest_last_analyzed]]
- :from [(t2/table-name :model/Field)]
+ :from [(warehouse-schema-overlay/field-query)]
:group-by [:table_id]} :sub]
[:= :t.id :sub.table_id]]
:where [:and sync-tables-clause [:= :t.db_id database-id]]
@@ -238,16 +247,19 @@
;;; ------------------------------------------------- Field -------------------------------------------------
(mu/defn fields
- "The Fields with `field-ids`."
+ "The Fields with `field-ids` as sync wrote them. Feeds `our-metadata`, which is diffed against the warehouse."
[field-ids :- [:sequential ::lib.schema.id/field]]
- (t2/select :model/Field :id [:in field-ids]))
+ (t2/select :model/Field :id [:in field-ids]
+ {:from [(warehouse-schema-overlay/field-query {:user-settings? false})]}))
(mu/defn fields-for-field-values
- "The columns needed to scan FieldValues of the Fields with `field-ids`."
+ "The columns needed to scan FieldValues of the Fields with `field-ids`, as users see them: the eligibility decision
+ (`visibility_type`/`has_field_values`) has to see what the user set."
[field-ids :- [:sequential ::lib.schema.id/field]]
(t2/select [:model/Field :name :id :base_type :effective_type :coercion_strategy :semantic_type :visibility_type
:table_id :has_field_values]
- :id [:in field-ids]))
+ :id [:in field-ids]
+ {:from [(warehouse-schema-overlay/field-query)]}))
(defn- base-types->descendants
"Given a set of `base-types`, an expanded set including those types and all their descendants in the type
@@ -302,7 +314,8 @@
version->base-types :- [:map-of :int [:set [:or :keyword :string]]]
limit :- ms/PositiveInt]
(t2/select :model/Field
- {:where [:and
+ {:from [(warehouse-schema-overlay/field-query)]
+ :where [:and
[:= :table_id table-id]
(needs-fingerprint-update-clause refingerprint? version->base-types)]
:order-by [[:id :asc]]
@@ -311,10 +324,10 @@
(mu/defn field-fingerprint
"The fingerprint of the Field with `field-id`."
[field-id :- ::lib.schema.id/field]
- (t2/select-one-fn :fingerprint :model/Field :id field-id))
+ (t2/select-one-fn :fingerprint :model/Field :id field-id {:from [(warehouse-schema-overlay/field-query {:user-settings? false})]}))
(mu/defn active-fields-metadata-for-table
- "The sync metadata columns of the active Fields of the Table with `table-id`, in field order."
+ "The sync metadata columns of the active Fields of the Table with `table-id`, as sync wrote them, in field order."
[table-id :- ::lib.schema.id/table]
(t2/select [:model/Field :name :database_type :base_type :effective_type :coercion_strategy :semantic_type
:parent_id :id :description :database_position :nfc_path
@@ -323,7 +336,8 @@
:database_partitioned :json_unfolding :position :preview_display]
:table_id table-id
:active true
- {:order-by table/field-order-rule}))
+ {:from [(warehouse-schema-overlay/field-query {:user-settings? false})]
+ :order-by table/field-order-rule}))
(mu/defn normal-fields-for-table
"Up to `limit` active, normal-visibility Fields of the Table with `table-id`, ordered by ID."
@@ -333,7 +347,8 @@
:table_id table-id
:active true
:visibility_type "normal"
- {:order-by [[:id :asc]], :limit limit}))
+ {:from [(warehouse-schema-overlay/field-query)]
+ :order-by [[:id :asc]], :limit limit}))
(mu/defn inactive-fields-by-lower-name
"The inactive Fields of the Table with `table-id` under `parent-id` whose lower-cased name is one of `lower-names`."
@@ -344,7 +359,7 @@
:table_id table-id
:%lower.name [:in lower-names]
:parent_id parent-id
- :active false))
+ :active false {:from [(warehouse-schema-overlay/field-query)]}))
(mu/defn incomplete-analysis-fields-for-table
"The active, visible Fields of the Table with `table-id` fingerprinted at `fingerprint-version` but not yet analyzed."
@@ -355,7 +370,8 @@
:active true
:visibility_type [:not-in ["sensitive" "retired"]]
:fingerprint_version fingerprint-version
- :last_analyzed nil))
+ :last_analyzed nil
+ {:from [(warehouse-schema-overlay/field-query)]}))
(mu/defn name-field-count-for-table
"The number of active, visible Fields of the Table with `table-id` whose semantic type is `:type/Name`."
@@ -364,25 +380,27 @@
:table_id table-id
:active true
:visibility_type [:not-in ["sensitive" "retired"]]
- :semantic_type :type/Name))
+ :semantic_type :type/Name
+ {:from [(warehouse-schema-overlay/field-query)]}))
(mu/defn unscored-fields-for-database-reducible
"Reducible active, visible Fields of the Database with `database-id` without a dimension interestingness score."
[database-id :- ::lib.schema.id/database]
(t2/reducible-select :model/Field
- {:where [:and
+ {:from [(warehouse-schema-overlay/field-query)]
+ :where [:and
[:= :active true]
[:= :dimension_interestingness nil]
[:not-in :visibility_type ["sensitive" "retired"]]
[:in :table_id ^:allow-subquery {:select [:id]
- :from [(t2/table-name :model/Table)]
+ :from [(warehouse-schema-overlay/table-query {:user-settings? false})]
:where [:= :db_id database-id]}]]}))
(mu/defn top-level-field-ids-by-name
"The IDs of the top-level Fields of the Table with `table-id` named one of `field-names`."
[table-id :- ::lib.schema.id/table
field-names :- [:sequential :string]]
- (t2/select-pks-vec :model/Field :name [:in field-names] :table_id table-id :parent_id nil))
+ (t2/select-pks-vec :model/Field :name [:in field-names] :table_id table-id :parent_id nil {:from [(warehouse-schema-overlay/field-query)]}))
(mu/defn top-level-field-ids-by-schema-table-and-name-reducible
"Reducible `:id` rows of the top-level Fields of the Database with `database-id` matching one of the
@@ -390,8 +408,8 @@
[database-id :- ::lib.schema.id/database
schema+table+names :- [:sequential [:tuple :string :string :string]]]
(t2/reducible-query {:select [[:f.id]]
- :from [[(t2/table-name :model/Field) :f]]
- :inner-join [[(t2/table-name :model/Table) :t] [:= :f.table_id :t.id]]
+ :from [(warehouse-schema-overlay/field-query {:alias :f})]
+ :inner-join [(warehouse-schema-overlay/table-query {:alias :t}) [:= :f.table_id :t.id]]
:where [:and
[:in [:composite [:coalesce :t.schema "__null__"] :t.name :f.name] schema+table+names]
[:= :t.db_id database-id]
@@ -400,14 +418,15 @@
(mu/defn indexed-field-ids-for-table
"The IDs of the Fields of the Table with `table-id` marked as indexed."
[table-id :- ::lib.schema.id/table]
- (t2/select-pks-set :model/Field :table_id table-id :database_indexed true))
+ (t2/select-pks-set :model/Field :table_id table-id :database_indexed true {:from [(warehouse-schema-overlay/field-query {:user-settings? false})]}))
(mu/defn indexed-top-level-field-ids-for-database
"The IDs of the top-level Fields of the Database with `database-id` marked as indexed."
[database-id :- ::lib.schema.id/database]
(t2/select-pks-set :model/Field
+ {:from [(warehouse-schema-overlay/field-query)]}
:table_id [:in ^:allow-subquery {:select [[:t.id]]
- :from [[(t2/table-name :model/Table) :t]]
+ :from [(warehouse-schema-overlay/table-query {:alias :t})]
:where [:= :t.db_id database-id]}]
:parent_id nil
:database_indexed true))
@@ -488,8 +507,8 @@
[db-id table-schema table-name column-name]
^:allow-subquery
{:select [[[:min :f.id] :id]]
- :from [[:metabase_field :f]]
- :join [[:metabase_table :t] [:= :f.table_id :t.id]]
+ :from [(warehouse-schema-overlay/field-query {:alias :f})]
+ :join [(warehouse-schema-overlay/table-query {:alias :t}) [:= :f.table_id :t.id]]
:left-join [[:metabase_field_user_settings :u] [:= :f.id :u.field_id]]
:where [:and
[:= :u.fk_target_field_id nil]
@@ -519,7 +538,7 @@
pk-field-id-query (fk-field-id-subquery db-id pk-table-schema pk-table-name pk-column-name)
q (case (app-db/db-type)
:mysql
- {:update [:metabase_field :f]
+ {:update [(t2/table-name :model/Field) :f]
:join [[fk-field-id-query :fk] [:= :fk.id :f.id]
[pk-field-id-query :pk]
(fk-target-changed-clause :pk.id)]
@@ -528,7 +547,7 @@
:has_field_values [:case [:= :has_field_values "auto-list"] nil :else :has_field_values]
:semantic_type "type/FK"}}
:postgres
- {:update [:metabase_field :f]
+ {:update [(t2/table-name :model/Field) :f]
:from [[fk-field-id-query :fk]]
:join [[pk-field-id-query :pk] true]
:set {:fk_target_field_id :pk.id
@@ -539,7 +558,7 @@
[:= :fk.id :f.id]
(fk-target-changed-clause :pk.id)]}
:h2
- {:update [:metabase_field :f]
+ {:update [(t2/table-name :model/Field) :f]
:set {:fk_target_field_id pk-field-id-query
;; We need to reset has_field_values when it is auto-list as FKs should not be marked as such
:has_field_values [:case [:= :has_field_values "auto-list"] nil :else :has_field_values]
@@ -575,12 +594,13 @@
[:= :data_sensitivity nil]))
(mu/defn fields-to-scan-for-data-sensitivity
- "The active, non-retired Fields of the Table with `table-id` that the data-sensitivity classifier still has to scan
- (see [[data-sensitivity-to-scan-clause]]), ordered by ID."
+ "The active, non-retired Fields of the Table with `table-id` the data-sensitivity classifier still has to scan, by
+ sync's own labels, ordered by ID."
[table-id :- ::lib.schema.id/table
rescan-public? :- [:maybe :boolean]]
(t2/select :model/Field
- {:where [:and
+ {:from [(warehouse-schema-overlay/field-query {:user-settings? false})]
+ :where [:and
[:= :table_id table-id]
[:= :active true]
[:not= :visibility_type "retired"]
@@ -594,8 +614,8 @@
rescan-public? :- [:maybe :boolean]]
(t2/select-fn-set :table_id :model/Field
{:select [[:metabase_field.table_id :table_id]]
- :from [:metabase_field]
- :join [[:metabase_table] [:= :metabase_field.table_id :metabase_table.id]]
+ :from [(warehouse-schema-overlay/field-query {:user-settings? false})]
+ :join [(warehouse-schema-overlay/table-query {:user-settings? false}) [:= :metabase_field.table_id :metabase_table.id]]
:where [:and
[:= :metabase_table.db_id database-id]
[:= :metabase_table.active true]
@@ -607,7 +627,8 @@
(mu/defn tables-by-schema-and-name-reducible
"Reducible Tables with `table-ids`, ordered by schema and name."
[table-ids :- [:set ::lib.schema.id/table]]
- (t2/reducible-select :model/Table :id [:in table-ids] {:order-by [[:schema :asc] [:name :asc]]}))
+ (t2/reducible-select :model/Table :id [:in table-ids] {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]
+ :order-by [[:schema :asc] [:name :asc]]}))
(mu/defn update-field-data-sensitivity!
"Set the `data_sensitivity` of the Field with `field-id` to `data-sensitivity`, returning the number updated."
diff --git a/src/metabase/sync/sync_metadata/fields/sync_instances.clj b/src/metabase/sync/sync_metadata/fields/sync_instances.clj
index 626b49e726e..92036b162c5 100644
--- a/src/metabase/sync/sync_metadata/fields/sync_instances.clj
+++ b/src/metabase/sync/sync_metadata/fields/sync_instances.clj
@@ -9,7 +9,6 @@
(:require
[medley.core :as m]
[metabase.lib.schema.id :as lib.schema.id]
- [metabase.models.humanization :as humanization]
[metabase.sync.db :as sync.db]
[metabase.sync.interface :as i]
[metabase.sync.sync-metadata.fields.common :as common]
@@ -18,7 +17,8 @@
[metabase.util :as u]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
- [metabase.util.malli.schema :as ms]))
+ [metabase.util.malli.schema :as ms]
+ [metabase.warehouse-schema.humanization :as warehouse-schema.humanization]))
;;; +----------------------------------------------------------------------------------------------------------------+
;;; | CREATING / REACTIVATING FIELDS |
@@ -89,7 +89,7 @@
base-type)))
{:table_id (u/the-id table)
:name field-name
- :display_name (humanization/name->human-readable-name field-name)
+ :display_name (warehouse-schema.humanization/name->human-readable-name field-name)
:database_type (or database-type "NULL") ; placeholder for Fields w/ no type info (e.g. Mongo) & all NULL
:base_type base-type
;; todo test this?
diff --git a/src/metabase/sync/sync_metadata/fields/sync_metadata.clj b/src/metabase/sync/sync_metadata/fields/sync_metadata.clj
index de54162f528..1fc75e138b5 100644
--- a/src/metabase/sync/sync_metadata/fields/sync_metadata.clj
+++ b/src/metabase/sync/sync_metadata/fields/sync_metadata.clj
@@ -13,7 +13,7 @@
[metabase.util.log :as log]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
- [metabase.warehouse-schema.models.field-user-settings :as schema.field-user-settings]))
+ [metabase.warehouse-schema.models.field-user-settings :as field-user-settings]))
(defn- normalize-nfc-path
"Normalize a `nfc-path` to a vector of strings so a driver emitting keywords doesn't churn against the
@@ -84,7 +84,6 @@
new-base-type?
(not= old-base-type new-base-type)
- ;; only sync comment if old value was blank so we don't overwrite user-set values
new-semantic-type?
(and (nil? old-semantic-type)
(not= old-semantic-type new-semantic-type))
@@ -128,22 +127,16 @@
(common/field-metadata-name-for-logging table metabase-field)
old-base-type
new-base-type)
- (doto
- {:base_type new-base-type
- :effective_type new-base-type
- :coercion_strategy nil
- ;; reset fingerprint version so this field will get re-fingerprinted and analyzed
- :fingerprint_version 0
- :fingerprint nil
- ;; semantic type needs to be set to nil so that the fingerprinter can re-infer it during analysis
- :semantic_type nil}
- ;; we must override user-set values
- (->> (schema.field-user-settings/upsert-user-settings metabase-field))))
- ;; GHY-3388 self-heal: a Field with no coercion_strategy must have effective_type=base_type.
- ;; We've observed customer instances where these drifted apart (likely from older Metabase
- ;; versions). When base_type didn't change at this sync but the row is in the broken state,
- ;; repair it. Wipe user-settings's stale effective_type too so sync-user-settings's merge-back
- ;; doesn't re-introduce drift on the next field update.
+ (field-user-settings/unset-user-settings!
+ metabase-field [:effective_type :coercion_strategy :semantic_type])
+ {:base_type new-base-type
+ :effective_type new-base-type
+ :coercion_strategy nil
+ ;; reset fingerprint version so this field will get re-fingerprinted and analyzed
+ :fingerprint_version 0
+ :fingerprint nil
+ ;; semantic type needs to be set to nil so that the fingerprinter can re-infer it during analysis
+ :semantic_type nil})
(when (and (not new-base-type?)
(nil? old-coercion-strategy)
(some? old-effective-type)
@@ -152,9 +145,8 @@
(common/field-metadata-name-for-logging table metabase-field)
old-effective-type
new-base-type)
- (let [et {:effective_type new-base-type}]
- (schema.field-user-settings/upsert-user-settings metabase-field et)
- et))
+ (field-user-settings/unset-user-settings! metabase-field [:effective_type :coercion_strategy])
+ {:effective_type new-base-type})
(when new-semantic-type?
(log/infof "Semantic type of %s has changed from '%s' to '%s'."
(common/field-metadata-name-for-logging table metabase-field)
diff --git a/src/metabase/sync/sync_metadata/tables.clj b/src/metabase/sync/sync_metadata/tables.clj
index 91051bd84ab..8711d948f68 100644
--- a/src/metabase/sync/sync_metadata/tables.clj
+++ b/src/metabase/sync/sync_metadata/tables.clj
@@ -9,7 +9,6 @@
[metabase.driver.util :as driver.u]
[metabase.lib.schema.common :as lib.schema.common]
[metabase.lib.schema.id :as lib.schema.id]
- [metabase.models.humanization :as humanization]
[metabase.models.interface :as mi]
[metabase.sync.db :as sync.db]
[metabase.sync.fetch-metadata :as fetch-metadata]
@@ -20,7 +19,8 @@
[metabase.util :as u]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
- [metabase.util.malli.schema :as ms]))
+ [metabase.util.malli.schema :as ms]
+ [metabase.warehouse-schema.humanization :as warehouse-schema.humanization]))
(set! *warn-on-reflection* true)
@@ -138,7 +138,7 @@
:description (:description table)
:database_require_filter (:database_require_filter table)
:display_name (or (:display_name table)
- (humanization/name->human-readable-name (:name table)))
+ (warehouse-schema.humanization/name->human-readable-name (:name table)))
:name (:name table)
:is_writable (:is_writable table)}
(when (:field_order table)
diff --git a/src/metabase/transforms/db.clj b/src/metabase/transforms/db.clj
index 53e423f3d28..9428d505435 100644
--- a/src/metabase/transforms/db.clj
+++ b/src/metabase/transforms/db.clj
@@ -11,6 +11,7 @@
[metabase.util.honey-sql-2 :as h2x]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
(def ^:private no-active-run-clause
@@ -777,7 +778,7 @@
(mu/defn table
"The Table with `table-id`, or nil."
[table-id :- ::lib.schema.id/table]
- (t2/select-one :model/Table table-id))
+ (t2/select-one :model/Table :id table-id {:from [(warehouse-schema-overlay/table-query)]}))
(mu/defn databases
"The Databases with `database-ids`."
@@ -792,7 +793,7 @@
(mu/defn tables
"The Tables with `table-ids`."
[table-ids :- [:set ::lib.schema.id/table]]
- (t2/select :model/Table :id [:in table-ids]))
+ (t2/select :model/Table :id [:in table-ids] {:from [(warehouse-schema-overlay/table-query)]}))
(mu/defn table-indexes-for-transforms
"The TableIndexes of the Transforms with `transform-ids`, ordered by index name."
@@ -802,17 +803,17 @@
(mu/defn field
"The Field with `field-id`, or nil."
[field-id :- ::lib.schema.id/field]
- (t2/select-one :model/Field field-id))
+ (t2/select-one :model/Field :id field-id {:from [(warehouse-schema-overlay/field-query)]}))
(mu/defn field-exists?
"Whether a Field with `field-id` exists."
[field-id :- ::lib.schema.id/field]
- (t2/exists? :model/Field :id field-id))
+ (t2/exists? :model/Field :id field-id {:from [(warehouse-schema-overlay/field-query)]}))
(mu/defn active-field-ids-by-name
"A map of name to ID for the active Fields of the Table with `table-id`."
[table-id :- ::lib.schema.id/table]
- (t2/select-fn->fn :name :id [:model/Field :name :id] :table_id table-id :active true))
+ (t2/select-fn->fn :name :id [:model/Field :name :id] :table_id table-id :active true {:from [(warehouse-schema-overlay/field-query {:user-settings? false})]}))
(mu/defn active-users
"The active Users with `user-ids`."
diff --git a/src/metabase/transforms_base/db.clj b/src/metabase/transforms_base/db.clj
index ae3549e91bb..f4fcb25acf9 100644
--- a/src/metabase/transforms_base/db.clj
+++ b/src/metabase/transforms_base/db.clj
@@ -5,6 +5,7 @@
[metabase.lib.schema.id :as lib.schema.id]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
(mu/defn transform-source
@@ -49,7 +50,7 @@
(mu/defn field-table-id
"The Table id of the Field with `field-id`, or nil."
[field-id :- ::lib.schema.id/field]
- (t2/select-one-fn :table_id :model/Field field-id))
+ (t2/select-one-fn :table_id :model/Field :id field-id {:from [(warehouse-schema-overlay/field-query {:user-settings? false})]}))
(mu/defn target-table
"The Table named `table-name` in `schema` of the Database with `database-id` also matching the key-value
@@ -63,12 +64,12 @@
(mu/defn table
"The Table with `table-id`, or nil."
[table-id :- ::lib.schema.id/table]
- (t2/select-one :model/Table table-id))
+ (t2/select-one :model/Table :id table-id {:from [(warehouse-schema-overlay/table-query)]}))
(mu/defn table-for-transform
"The Table owned by the Transform with `transform-id`, or nil."
[transform-id :- ::lib.schema.id/transform]
- (t2/select-one :model/Table :transform_id transform-id))
+ (t2/select-one :model/Table :transform_id transform-id {:from [(warehouse-schema-overlay/table-query)]}))
(mu/defn update-table!
"Apply `changes` to the Table with `table-id`."
@@ -106,14 +107,16 @@
[table-id :- ::lib.schema.id/table]
(t2/select-fn-vec :name [:model/Field :name :position]
:table_id table-id :active true
- {:order-by [[:position :asc]]}))
+ {:from [(warehouse-schema-overlay/field-query {:user-settings? false})]
+ :order-by [[:position :asc]]}))
(mu/defn table-refs-matching
"The id, Database id, schema, and name of the Tables matching any of `refs` (each a `[db-id schema table-name]`
triple; `schema` may be nil)."
[refs :- [:sequential [:tuple ms/PositiveInt [:maybe :string] :string]]]
(t2/select [:model/Table :id :db_id :schema :name]
- {:where (into [:or]
+ {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]
+ :where (into [:or]
(map (fn [[db-id schema table-name]]
[:and
[:= :db_id db-id]
@@ -126,4 +129,4 @@
(mu/defn table-refs
"The id, Database id, schema, and name of the Tables with `table-ids`."
[table-ids :- [:set ::lib.schema.id/table]]
- (t2/select [:model/Table :id :db_id :schema :name] :id [:in table-ids]))
+ (t2/select [:model/Table :id :db_id :schema :name] :id [:in table-ids] {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))
diff --git a/src/metabase/typed_schemas/db.clj b/src/metabase/typed_schemas/db.clj
index b16eae76ff9..0152166d75f 100644
--- a/src/metabase/typed_schemas/db.clj
+++ b/src/metabase/typed_schemas/db.clj
@@ -6,6 +6,7 @@
[metabase.lib.schema.id :as lib.schema.id]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
(mu/defn destination-database-ids
@@ -41,7 +42,7 @@
(mu/defn field-ids-and-table-ids
"The id and Table id of the Fields with `field-ids`."
[field-ids :- [:set ::lib.schema.id/field]]
- (t2/select [:model/Field :id :table_id] :id [:in field-ids]))
+ (t2/select [:model/Field :id :table_id] :id [:in field-ids] {:from [(warehouse-schema-overlay/field-query {:user-settings? false})]}))
(mu/defn card-dimensions
"The dimensions and dimension mappings of the Card with `card-id`, or nil."
@@ -51,7 +52,7 @@
(mu/defn table-names
"The id, name, and display name of the Tables with `table-ids`."
[table-ids :- [:sequential ::lib.schema.id/table]]
- (t2/select [:model/Table :id :name :display_name] :id [:in table-ids]))
+ (t2/select [:model/Table :id :name :display_name] :id [:in table-ids] {:from [(warehouse-schema-overlay/table-query)]}))
(mu/defn model-actions
"The id, model id, name, and type of the unarchived non-HTTP Actions of the model Cards with `model-ids`."
@@ -64,14 +65,15 @@
(mu/defn field-table-id
"The Table id of the Field with `field-id`, or nil."
[field-id :- ::lib.schema.id/field]
- (t2/select-one-fn :table_id :model/Field :id field-id))
+ (t2/select-one-fn :table_id :model/Field :id field-id {:from [(warehouse-schema-overlay/field-query {:user-settings? false})]}))
(mu/defn active-tables-in-scope
"The active Tables among `database-ids` and/or `table-ids` (either nil for unscoped), in name then id order."
[database-ids :- [:maybe [:set ::lib.schema.id/database]]
table-ids :- [:maybe [:or [:set ::lib.schema.id/table] [:sequential ::lib.schema.id/table]]]]
(t2/select :model/Table
- {:where [:and [:= :active true]
+ {:from [(warehouse-schema-overlay/table-query)]
+ :where [:and [:= :active true]
(scope-filter-clause database-ids :db_id)
(scope-filter-clause table-ids :id)]
:order-by [[:name :asc] [:id :asc]]}))
@@ -80,7 +82,8 @@
"The active, published Tables in `collection-ids` (nil for unscoped), in name then id order."
[collection-ids :- [:maybe [:set ::lib.schema.id/collection]]]
(t2/select :model/Table
- {:where [:and
+ {:from [(warehouse-schema-overlay/table-query)]
+ :where [:and
[:= :active true]
[:= :is_published true]
(scope-filter-clause collection-ids :collection_id)]
diff --git a/src/metabase/upload/db.clj b/src/metabase/upload/db.clj
index 1edf2e9b837..3f0db76c11b 100644
--- a/src/metabase/upload/db.clj
+++ b/src/metabase/upload/db.clj
@@ -3,9 +3,11 @@
additional logic, so the rest of the module only touches `toucan2.core` for hydration."
(:require
[metabase.lib.schema.id :as lib.schema.id]
- [metabase.models.humanization :as humanization]
[metabase.queries.schema :as queries.schema]
[metabase.util.malli :as mu]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
+ [metabase.warehouse-schema.core :as warehouse-schema]
+ [metabase.warehouse-schema.humanization :as humanization]
[toucan2.core :as t2]))
(mu/defn current-database
@@ -42,29 +44,37 @@
(mu/defn active-fields-for-table
"The active Fields of the Table with `table-id`."
[table-id :- ::lib.schema.id/table]
- (t2/select :model/Field :table_id table-id :active true))
+ (t2/select :model/Field :table_id table-id :active true {:from [(warehouse-schema-overlay/field-query)]}))
+
+(mu/defn user-edited-field-names
+ "The lower-cased `names`, among the Fields of the Table with `table-id`, that have a user-set display name."
+ [table-id :- ::lib.schema.id/table
+ names :- [:set :string]]
+ (warehouse-schema/user-renamed-field-names table-id names))
(mu/defn set-field-display-names!
"Set the display name of each Field of the Table with `table-id` whose lower-cased name is a key of
- `name->display-name` to the corresponding value, unless its current display name no longer matches the
- automatic humanization of its name (i.e. it was manually edited, in which case it is left alone)."
+ `name->display-name` to the corresponding value, unless the Field has a user-set display name recorded in
+ FieldUserSettings or its display name is not the automatic humanization of its name (in which case it is left
+ alone)."
[table-id :- ::lib.schema.id/table
name->display-name :- [:map-of :string :string]]
- ;; A raw update rather than `t2/update!`, which produces an invalid query for certain versions of PostgreSQL:
- ;; SELECT * FROM "metabase_field" WHERE "id" AND ("table_id" = ?) AND ... (argument of AND must be type boolean).
- (t2/query {:update (t2/table-name :model/Field)
- :set {:display_name (into [:case]
- (mapcat identity)
- (for [[n display-name] name->display-name]
- [[:= [:lower :name] n]
- [:case
- ;; Only update the display name if it still matches the automatic humanization.
- [:= :display_name (humanization/name->human-readable-name n)] display-name
- ;; Otherwise, it could have been set manually, so leave it as is.
- true :display_name]]))}
- :where [:and
- [:= :table_id table-id]
- [:in [:lower :name] (keys name->display-name)]]}))
+ (let [user-edited (user-edited-field-names table-id (set (keys name->display-name)))
+ name->display-name (apply dissoc name->display-name user-edited)]
+ (when (seq name->display-name)
+ ;; A raw update rather than `t2/update!`, which produces an invalid query for certain versions of PostgreSQL:
+ ;; SELECT * FROM "metabase_field" WHERE "id" AND ("table_id" = ?) AND ... (argument of AND must be type boolean).
+ (t2/query {:update (t2/table-name :model/Field)
+ :set {:display_name (into [:case]
+ (mapcat identity)
+ (for [[n display-name] name->display-name]
+ [[:= [:lower :name] n]
+ [:case
+ [:= :display_name (humanization/name->human-readable-name n)] display-name
+ :else :display_name]]))}
+ :where [:and
+ [:= :table_id table-id]
+ [:in [:lower :name] (keys name->display-name)]]}))))
(mu/defn mark-table-upload!
"Flag the Table with `table-id` as an authoritative, writable upload table."
@@ -112,9 +122,9 @@
(mu/defn table
"The Table with `table-id`, or nil."
[table-id :- ::lib.schema.id/table]
- (t2/select-one :model/Table :id table-id))
+ (t2/select-one :model/Table :id table-id {:from [(warehouse-schema-overlay/table-query)]}))
(mu/defn tables
"The Tables with `table-ids`."
[table-ids :- [:set ::lib.schema.id/table]]
- (t2/select :model/Table :id [:in table-ids]))
+ (t2/select :model/Table :id [:in table-ids] {:from [(warehouse-schema-overlay/table-query)]}))
diff --git a/src/metabase/upload/impl.clj b/src/metabase/upload/impl.clj
index 28e4da03e5c..7b5ced39679 100644
--- a/src/metabase/upload/impl.clj
+++ b/src/metabase/upload/impl.clj
@@ -20,7 +20,6 @@
[metabase.lib-be.schema :as lib-be.schema]
[metabase.lib.core :as lib]
[metabase.model-persistence.core :as model-persistence]
- [metabase.models.humanization :as humanization]
[metabase.models.interface :as mi]
[metabase.permissions.core :as perms]
[metabase.queries.core :as queries]
@@ -33,6 +32,7 @@
[metabase.util.i18n :refer [tru]]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
+ [metabase.warehouse-schema.humanization :as warehouse-schema.humanization]
[metabase.warehouse-schema.models.table :as table]
[toucan2.core :as t2])
(:import
@@ -370,7 +370,7 @@
(let [generator-fn (lib/unique-name-generator-with-options {:unique-alias-fn (unique-alias-fn driver " ")})]
(mapv generator-fn
(for [h header]
- (humanization/name->human-readable-name
+ (warehouse-schema.humanization/name->human-readable-name
(normalize-display-name h))))))
(defn- derive-column-names [driver header]
@@ -620,7 +620,7 @@
(let [timer (u/start-timer)
filename-prefix (or (second (re-matches #"(.*)\.(csv|tsv)$" filename))
filename)
- humanized-name (humanization/name->human-readable-name filename-prefix)
+ humanized-name (warehouse-schema.humanization/name->human-readable-name filename-prefix)
display-name (u/truncate-string-to-byte-count humanized-name (max-bytes :model/Table :display_name))
card-name (u/truncate-string-to-byte-count humanized-name (max-bytes :model/Card :name))
driver (driver.u/database->driver database)
@@ -776,7 +776,7 @@
(let [card (upload.db/card-query-and-metadata id)
;; Unclear why this is required, would expect it to get this from the field's display name, as it does for
;; the initial upload.
- fix-name #(update % :display_name humanization/name->human-readable-name)
+ fix-name #(update % :display_name warehouse-schema.humanization/name->human-readable-name)
metadata (queries/refresh-metadata card {:update-fn fix-name})]
(upload.db/set-card-result-metadata! id metadata)))))
diff --git a/src/metabase/usage_metadata/db.clj b/src/metabase/usage_metadata/db.clj
index ae31c62be7b..9c4ca49c012 100644
--- a/src/metabase/usage_metadata/db.clj
+++ b/src/metabase/usage_metadata/db.clj
@@ -7,6 +7,7 @@
[metabase.usage-metadata.schema :as usage-metadata.schema]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
(mu/defn query-execution-hash-counts
@@ -114,17 +115,17 @@
(mu/defn field-names
"The id, name, and display name of the Fields with `field-ids`."
[field-ids :- [:set ::lib.schema.id/field]]
- (t2/select [:model/Field :id :name :display_name] :id [:in field-ids]))
+ (t2/select [:model/Field :id :name :display_name] :id [:in field-ids] {:from [(warehouse-schema-overlay/field-query)]}))
(mu/defn table-names
"The id, name, display name, Database id, and schema of the Tables with `table-ids`."
[table-ids :- [:set ::lib.schema.id/table]]
- (t2/select [:model/Table :id :name :display_name :db_id :schema] :id [:in table-ids]))
+ (t2/select [:model/Table :id :name :display_name :db_id :schema] :id [:in table-ids] {:from [(warehouse-schema-overlay/table-query)]}))
(mu/defn table-database-ids
"The id and Database id of the Tables with `table-ids`."
[table-ids :- [:set ::lib.schema.id/table]]
- (t2/select [:model/Table :id :db_id] :id [:in table-ids]))
+ (t2/select [:model/Table :id :db_id] :id [:in table-ids] {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))
(mu/defn card-names
"The id and name of the Cards with `card-ids`."
diff --git a/src/metabase/warehouse_schema/core.clj b/src/metabase/warehouse_schema/core.clj
new file mode 100644
index 00000000000..f1a592f30a4
--- /dev/null
+++ b/src/metabase/warehouse_schema/core.clj
@@ -0,0 +1,12 @@
+(ns metabase.warehouse-schema.core
+ "API namespace for the warehouse-schema module."
+ (:require
+ [metabase.warehouse-schema.db]
+ [potemkin :as p]))
+
+(comment
+ metabase.warehouse-schema.db/keep-me)
+
+(p/import-vars
+ [metabase.warehouse-schema.db
+ user-renamed-field-names])
diff --git a/src/metabase/warehouse_schema/db.clj b/src/metabase/warehouse_schema/db.clj
index ee93b828fd2..3961f99af85 100644
--- a/src/metabase/warehouse_schema/db.clj
+++ b/src/metabase/warehouse_schema/db.clj
@@ -7,8 +7,10 @@
[metabase.app-db.core :as app-db]
[metabase.lib.schema.id :as lib.schema.id]
[metabase.models.db :as models.db]
+ [metabase.util :as u]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[metabase.warehouse-schema.schema :as warehouse-schema.schema]
[toucan2.core :as t2]))
@@ -16,12 +18,10 @@
"How should we order fields."
[[:position :asc] [:%lower.name :asc]])
-;;; ------------------------------------------------- ::warehouse-schema.schema/field -------------------------------------------------
-
(mu/defn field
"The ::warehouse-schema.schema/field with `field-id`, or nil."
[field-id :- ::lib.schema.id/field]
- (t2/select-one :model/Field :id field-id))
+ (t2/select-one :model/Field :id field-id {:from [(warehouse-schema-overlay/field-query)]}))
(mu/defn field-in-path
"The ::warehouse-schema.schema/field named by the last of `field-names` (each nested inside the previous, bottom-most first) under
@@ -33,44 +33,46 @@
(mu/defn fields
"The Fields with `field-ids`."
[field-ids :- [:set ::lib.schema.id/field]]
- (t2/select :model/Field :id [:in field-ids]))
+ (t2/select :model/Field :id [:in field-ids] {:from [(warehouse-schema-overlay/field-query)]}))
(mu/defn fields-by-id
"A map of ID to ::warehouse-schema.schema/field for `field-ids`."
[field-ids :- [:sequential ::lib.schema.id/field]]
- (t2/select-fn->fn :id identity :model/Field :id [:in field-ids]))
+ (t2/select-fn->fn :id identity :model/Field :id [:in field-ids] {:from [(warehouse-schema-overlay/field-query)]}))
(mu/defn field-table-id-rows
"The ID and ::warehouse-schema.schema/table ID of the Fields with `field-ids`."
[field-ids :- [:sequential ::lib.schema.id/field]]
- (t2/select [:model/Field :id :table_id] :id [:in field-ids]))
+ (t2/select [:model/Field :id :table_id] :id [:in field-ids] {:from [(warehouse-schema-overlay/field-query {:user-settings? false})]}))
(mu/defn field-table-id
"The ::warehouse-schema.schema/table ID of the ::warehouse-schema.schema/field with `field-id`."
[field-id :- ::lib.schema.id/field]
- (t2/select-one-fn :table_id :model/Field :id field-id))
+ (t2/select-one-fn :table_id :model/Field :id field-id {:from [(warehouse-schema-overlay/field-query {:user-settings? false})]}))
(mu/defn field-id-by-name
"The ID of the ::warehouse-schema.schema/field named `field-name` under `parent-id` in the ::warehouse-schema.schema/table with `table-id`, or nil."
[table-id :- ::lib.schema.id/table
parent-id :- [:maybe ms/PositiveInt]
field-name :- :string]
- (t2/select-one-pk :model/Field :name field-name :parent_id parent-id :table_id table-id))
+ (t2/select-one-pk :model/Field :name field-name :parent_id parent-id :table_id table-id {:from [(warehouse-schema-overlay/field-query {:user-settings? false})]}))
(mu/defn field-values-eligibility
"The columns deciding whether the ::warehouse-schema.schema/field with `field-id` should have FieldValues, or nil."
[field-id :- ::lib.schema.id/field]
- (t2/select-one [:model/Field :base_type :visibility_type :has_field_values :preview_display] :id field-id))
+ (t2/select-one [:model/Field :base_type :visibility_type :has_field_values :preview_display] :id field-id
+ {:from [(warehouse-schema-overlay/field-query)]}))
(mu/defn field-ids-for-table
"The IDs of the Fields of the ::warehouse-schema.schema/table with `table-id`."
[table-id :- ::lib.schema.id/table]
- (t2/select-pks-set :model/Field {:where [:= :table_id table-id]}))
+ (t2/select-pks-set :model/Field {:from [(warehouse-schema-overlay/field-query {:user-settings? false})]
+ :where [:= :table_id table-id]}))
(mu/defn active-field-ids-for-table
"The IDs of the active Fields of the ::warehouse-schema.schema/table with `table-id`."
[table-id :- ::lib.schema.id/table]
- (t2/select-pks-set :model/Field :table_id table-id :active true))
+ (t2/select-pks-set :model/Field :table_id table-id :active true {:from [(warehouse-schema-overlay/field-query {:user-settings? false})]}))
(defn- field-order-order-by
[field-order]
@@ -91,7 +93,8 @@
or `:alphabetical`)."
[table-id :- ::lib.schema.id/table
field-order :- [:enum :custom :smart :database :alphabetical]]
- (t2/select [:model/Field :id] :table_id table-id {:order-by (field-order-order-by field-order)}))
+ (t2/select [:model/Field :id] :table_id table-id {:from [(warehouse-schema-overlay/field-query)]
+ :order-by (field-order-order-by field-order)}))
(mu/defn active-fields-for-tables
"The active, unretired Fields of the Tables with `table-ids`, in field order."
@@ -100,7 +103,8 @@
:active true
:table_id [:in table-ids]
:visibility_type [:not= "retired"]
- {:order-by field-order-rule}))
+ {:from [(warehouse-schema-overlay/field-query)]
+ :order-by field-order-rule}))
(mu/defn pk-field-ids-by-table
"A map of ::warehouse-schema.schema/table ID to the ID of its visible primary key ::warehouse-schema.schema/field for `table-ids`."
@@ -108,23 +112,13 @@
(t2/select-fn->fn :table_id :id :model/Field
:table_id [:in table-ids]
:semantic_type (app-db/isa :type/PK)
- :visibility_type [:not-in ["sensitive" "retired"]]))
-
-(mu/defn fk-source-field-ids-without-user-settings
- "The `:id` rows of the Fields targeting the ::warehouse-schema.schema/field with `field-id` that have no FieldUserSettings row."
- [field-id :- ::lib.schema.id/field]
- (t2/query {:select [:id]
- :from [:metabase_field]
- :where [:and
- [:= :fk_target_field_id field-id]
- [:not [:exists ^:allow-subquery {:select [1]
- :from [:metabase_field_user_settings]
- :where [:= :metabase_field_user_settings.field_id :metabase_field.id]}]]]}))
+ :visibility_type [:not-in ["sensitive" "retired"]]
+ {:from [(warehouse-schema-overlay/field-query {:alias :f})]}))
(mu/defn update-field!
"Apply `changes` to the ::warehouse-schema.schema/field with `field-id`, returning the number updated."
[field-id :- ::lib.schema.id/field
- changes :- (mut/select-keys ::warehouse-schema.schema/field.update [:position :custom_position])]
+ changes :- (mut/select-keys ::warehouse-schema.schema/field.update [:position])]
(t2/update! :model/Field field-id changes))
(mu/defn clear-fk-targets-to-field!
@@ -143,6 +137,37 @@
[table-id :- ::lib.schema.id/table]
(t2/delete! :model/Field :table_id table-id))
+(mu/defn user-renamed-field-names :- [:set :string]
+ "The lower-cased names, among `names`, of the Fields of the Table with `table-id` whose display name a user set.
+ Uploads use it to leave those Fields alone when appending re-derives display names from the CSV header."
+ [table-id :- ::lib.schema.id/table
+ names :- [:set :string]]
+ (set
+ (t2/select-fn-set (comp u/lower-case-en :name)
+ :model/Field
+ {:select [:f.name]
+ :from [(warehouse-schema-overlay/field-query {:alias :f, :user-settings? false})]
+ :left-join [[(t2/table-name :model/FieldUserSettings) :u] [:= :u.field_id :f.id]]
+ :where [:and
+ [:= :f.table_id table-id]
+ [:in [:lower :f.name] names]
+ [:not= :u.display_name nil]]})))
+
+(mu/defn field-names-reducible
+ "A reducible of the id, name, and display name of every ::warehouse-schema.schema/field, plus its user-set display
+ name from FieldUserSettings (if any) as `:user_display_name`."
+ []
+ (t2/reducible-query
+ {:select [:f.id :f.name :f.display_name [:u.display_name :user_display_name]]
+ :from [(warehouse-schema-overlay/field-query {:alias :f, :user-settings? false})]
+ :left-join [[(t2/table-name :model/FieldUserSettings) :u] [:= :u.field_id :f.id]]}))
+
+(mu/defn set-field-display-name!
+ "Set the display name of the ::warehouse-schema.schema/field with `id`, returning the number updated."
+ [id :- ::lib.schema.id/field
+ display-name :- :string]
+ (t2/update! :model/Field id {:display_name display-name}))
+
;;; -------------------------------------------- FieldUserSettings --------------------------------------------
(mu/defn field-user-settings
@@ -156,29 +181,143 @@
[field-id :- ::lib.schema.id/field]
(t2/exists? :model/FieldUserSettings field-id))
-(mu/defn user-edited-field-ids-for-table
- "The IDs of the Fields of the ::warehouse-schema.schema/table with `table-id` that have a FieldUserSettings row."
+(def ^:private field-user-settings-update-keys
+ "The columns an insert or update of a FieldUserSettings accepts."
+ [:field_id :created_at :updated_at :semantic_type :description :display_name :visibility_type :fk_target_field_id :has_field_values :effective_type :coercion_strategy :caveats :points_of_interest :nfc_path :json_unfolding :settings :data_sensitivity :custom_position :description_set :semantic_type_set :fk_target_field_id_set])
+
+(mu/defn field-user-settings-exist-for-table?
+ "Whether any Field of the ::warehouse-schema.schema/table with `table-id` has a FieldUserSettings row."
[table-id :- ::lib.schema.id/table]
- (t2/select-fn-set :field_id :model/FieldUserSettings
- {:join [[:metabase_field :f] [:= :f.id :field_id]]
- :where [:= :f.table_id table-id]}))
+ (t2/exists? :model/FieldUserSettings
+ {:from [[(t2/table-name :model/FieldUserSettings) :u]]
+ :join [(warehouse-schema-overlay/field-query {:alias :f, :user-settings? false}) [:= :f.id :u.field_id]]
+ :where [:= :f.table_id table-id]}))
+
+(mu/defn field-user-settings-for-tables
+ "The FieldUserSettings of the Fields of `table-ids`, each with its Field's `:table_id`, in Field name order."
+ [table-ids :- [:sequential ::lib.schema.id/table]]
+ (t2/select :model/FieldUserSettings
+ {:select [:u.* [:f.table_id :table_id] [:f.name :field_name]]
+ :from [[(t2/table-name :model/FieldUserSettings) :u]]
+ :join [(warehouse-schema-overlay/field-query {:alias :f, :user-settings? false}) [:= :f.id :u.field_id]]
+ :where [:in :f.table_id table-ids]
+ :order-by [[:f.name :asc]]}))
(mu/defn insert-field-user-settings!
"Insert one FieldUserSettings map or a sequence of them, returning the number inserted."
- [rows :- [:or (mut/select-keys ::warehouse-schema.schema/field-user-settings.update [:field_id :created_at :updated_at :semantic_type :description :display_name :visibility_type :fk_target_field_id :has_field_values :effective_type :coercion_strategy :caveats :points_of_interest :nfc_path :json_unfolding :settings :data_sensitivity]) [:sequential (mut/select-keys ::warehouse-schema.schema/field-user-settings.update [:field_id :created_at :updated_at :semantic_type :description :display_name :visibility_type :fk_target_field_id :has_field_values :effective_type :coercion_strategy :caveats :points_of_interest :nfc_path :json_unfolding :settings :data_sensitivity])]]]
+ [rows :- [:or (mut/select-keys ::warehouse-schema.schema/field-user-settings.update field-user-settings-update-keys) [:sequential (mut/select-keys ::warehouse-schema.schema/field-user-settings.update field-user-settings-update-keys)]]]
(t2/insert! :model/FieldUserSettings rows))
(mu/defn update-field-user-settings!
"Apply `changes` to the FieldUserSettings of the ::warehouse-schema.schema/field with `field-id`, returning the number updated."
[field-id :- ::lib.schema.id/field
- changes :- (mut/select-keys ::warehouse-schema.schema/field-user-settings.update [:field_id :created_at :updated_at :semantic_type :description :display_name :visibility_type :fk_target_field_id :has_field_values :effective_type :coercion_strategy :caveats :points_of_interest :nfc_path :json_unfolding :settings :data_sensitivity])]
+ changes :- (mut/select-keys ::warehouse-schema.schema/field-user-settings.update field-user-settings-update-keys)]
(t2/update! :model/FieldUserSettings field-id changes))
+(mu/defn field-ids-with-user-settings :- [:set ::lib.schema.id/field]
+ "The ids, among `field-ids`, of the Fields that have a FieldUserSettings row."
+ [field-ids :- [:sequential ::lib.schema.id/field]]
+ (set (t2/select-fn-set :field_id :model/FieldUserSettings :field_id [:in field-ids])))
+
+(mu/defn update-field-user-settings-custom-positions!
+ "Set the `custom_position` of the FieldUserSettings of each Field in `field-id->position`, returning the number
+ updated."
+ [field-id->position :- [:map-of ::lib.schema.id/field :int]]
+ (t2/update! :model/FieldUserSettings :field_id [:in (keys field-id->position)]
+ {:custom_position (into [:case] (mapcat (fn [[id position]] [[:= :field_id id] position])) field-id->position)}))
+
+(mu/defn delete-field-user-settings!
+ "Delete the FieldUserSettings of the ::warehouse-schema.schema/field with `field-id`, returning the number deleted."
+ [field-id :- ::lib.schema.id/field]
+ (t2/delete! :model/FieldUserSettings :field_id field-id))
+
(mu/defn clear-user-settings-fk-targets-to-field!
- "Clear the user-set FK semantic type and target of every ::warehouse-schema.schema/field targeting the ::warehouse-schema.schema/field with `field-id`, returning the
- number updated."
+ "Unset the user-set FK semantic type and target of every ::warehouse-schema.schema/field targeting the
+ ::warehouse-schema.schema/field with `field-id`, so the sync values show again. Returns the number updated."
[field-id :- ::lib.schema.id/field]
- (t2/update! :model/FieldUserSettings {:fk_target_field_id field-id} {:semantic_type nil, :fk_target_field_id nil}))
+ (t2/update! :model/FieldUserSettings {:fk_target_field_id field-id}
+ {:semantic_type nil, :semantic_type_set false, :fk_target_field_id nil, :fk_target_field_id_set false}))
+
+;;; -------------------------------------------- TableUserSettings --------------------------------------------
+
+(def ^:private table-user-settings-update-keys
+ "The columns an insert or update of a TableUserSettings accepts."
+ [:table_id :created_at :updated_at :display_name :description :entity_type :visibility_type :caveats
+ :points_of_interest :data_layer :data_source :owner_email :owner_user_id :field_order :show_in_getting_started
+ :data_authority :is_published :collection_id :description_set
+ :visibility_type_set :caveats_set :points_of_interest_set :data_layer_set :data_source_set])
+
+(mu/defn table-user-settings
+ "The TableUserSettings of the ::warehouse-schema.schema/table with `table-id`, or nil (also for a nil `table-id`,
+ e.g. a ::warehouse-schema.schema/table not yet inserted)."
+ [table-id :- [:maybe ::lib.schema.id/table]]
+ (t2/select-one :model/TableUserSettings :table_id table-id))
+
+(mu/defn table-user-settings-exist?
+ "Whether the ::warehouse-schema.schema/table with `table-id` has a TableUserSettings row."
+ [table-id :- ::lib.schema.id/table]
+ (t2/exists? :model/TableUserSettings table-id))
+
+(mu/defn table-ids-with-user-settings :- [:set ::lib.schema.id/table]
+ "The ids, among `table-ids`, of the Tables that already have a TableUserSettings row."
+ [table-ids :- [:set ::lib.schema.id/table]]
+ (set (t2/select-fn-set :table_id :model/TableUserSettings :table_id [:in table-ids])))
+
+(mu/defn update-table-user-settings-for-tables!
+ "Apply `changes` to the TableUserSettings of the Tables with `table-ids`, returning the number updated."
+ [table-ids :- [:set ::lib.schema.id/table]
+ changes :- (mut/select-keys ::warehouse-schema.schema/table-user-settings.update table-user-settings-update-keys)]
+ (t2/update! :model/TableUserSettings :table_id [:in table-ids] changes))
+
+(mu/defn insert-table-user-settings!
+ "Insert one TableUserSettings map or a sequence of them, returning the number inserted."
+ [rows :- [:or (mut/select-keys ::warehouse-schema.schema/table-user-settings.update table-user-settings-update-keys) [:sequential (mut/select-keys ::warehouse-schema.schema/table-user-settings.update table-user-settings-update-keys)]]]
+ (t2/insert! :model/TableUserSettings rows))
+
+(mu/defn update-table-user-settings!
+ "Apply `changes` to the TableUserSettings of the ::warehouse-schema.schema/table with `table-id`, returning the
+ number updated."
+ [table-id :- ::lib.schema.id/table
+ changes :- (mut/select-keys ::warehouse-schema.schema/table-user-settings.update table-user-settings-update-keys)]
+ (t2/update! :model/TableUserSettings table-id changes))
+
+(mu/defn delete-table-user-settings!
+ "Delete the TableUserSettings of the ::warehouse-schema.schema/table with `table-id`, returning the number deleted."
+ [table-id :- ::lib.schema.id/table]
+ (t2/delete! :model/TableUserSettings :table_id table-id))
+
+(mu/defn table-user-settings-with-field-settings
+ "One TableUserSettings per Table among `table-ids` (all when nil) that has a settings row or a Field with one,
+ synthesized as `{:table_id id}` when the Table has no row of its own."
+ [table-ids :- [:maybe [:sequential ::lib.schema.id/table]]]
+ (let [flag-columns (set (vals warehouse-schema-overlay/table-user-settings-flags))
+ value-columns (remove (some-fn #{:table_id} flag-columns) table-user-settings-update-keys)]
+ (t2/select
+ :model/TableUserSettings
+ {:select (into [[:t.id :table_id]]
+ (concat (map #(u/qualified-key :u %) value-columns)
+ (map (fn [flag] [[:coalesce (u/qualified-key :u flag) false] flag]) flag-columns)))
+ :from [(warehouse-schema-overlay/table-query {:alias :t, :user-settings? false})]
+ :left-join [[(t2/table-name :model/TableUserSettings) :u] [:= :u.table_id :t.id]]
+ :where [:and
+ (if table-ids [:in :t.id table-ids] true)
+ [:or
+ [:not= :u.table_id nil]
+ [:exists ^:allow-subquery
+ {:select [1]
+ :from [[(t2/table-name :model/FieldUserSettings) :fu]]
+ :join [(warehouse-schema-overlay/field-query {:alias :f, :user-settings? false}) [:= :f.id :fu.field_id]]
+ :where [:= :f.table_id :t.id]}]]]})))
+
+(mu/defn delete-field-user-settings-for-table!
+ "Delete the FieldUserSettings of the Fields of the ::warehouse-schema.schema/table with `table-id`, returning the
+ number deleted."
+ [table-id :- ::lib.schema.id/table]
+ (t2/delete! :model/FieldUserSettings
+ {:where [:exists ^:allow-subquery
+ {:select 1
+ :from [[(t2/table-name :model/Field) :f]]
+ :where [:and [:= :f.id :field_id] [:= :f.table_id table-id]]}]}))
;;; ---------------------------------------------- FieldValues ----------------------------------------------
@@ -203,7 +342,8 @@
"The ::warehouse-schema.schema/field ID, values, and ::warehouse-schema.schema/table ID of the full FieldValues of the normal Fields of the Tables with `table-ids`."
[table-ids :- [:sequential ::lib.schema.id/table]]
(t2/select [:model/FieldValues :field_id :values :field.table_id]
- {:join [[:metabase_field :field] [:= :metabase_fieldvalues.field_id :field.id]]
+ {:join [(warehouse-schema-overlay/field-query {:alias :field})
+ [:= :metabase_fieldvalues.field_id :field.id]]
:where [:and
[:in :field.table_id table-ids]
[:= :field.visibility_type "normal"]
@@ -280,40 +420,34 @@
(mu/defn table
"The ::warehouse-schema.schema/table with `table-id`, or nil."
[table-id :- [:maybe ::lib.schema.id/table]]
- (t2/select-one :model/Table :id table-id))
+ (t2/select-one :model/Table :id table-id {:from [(warehouse-schema-overlay/table-query)]}))
(mu/defn tables
"The Tables with `table-ids`."
[table-ids :- [:or [:set ::lib.schema.id/table] [:sequential ::lib.schema.id/table]]]
- (t2/select :model/Table :id [:in table-ids]))
+ (t2/select :model/Table :id [:in table-ids] {:from [(warehouse-schema-overlay/table-query)]}))
(mu/defn table-by-name
"The ::warehouse-schema.schema/table named `table-name` in `schema` of the ::warehouses.schema/database with `database-id`, or nil."
[database-id :- ::lib.schema.id/database
schema :- [:maybe :string]
table-name :- :string]
- (t2/select-one :model/Table :name table-name :db_id database-id :schema schema))
+ (t2/select-one :model/Table :name table-name :db_id database-id :schema schema {:from [(warehouse-schema-overlay/table-query)]}))
(mu/defn table-name-and-schema
"The name and schema of the ::warehouse-schema.schema/table with `table-id`."
[table-id :- ::lib.schema.id/table]
- (t2/select-one [:model/Table :name :schema] :id table-id))
+ (t2/select-one [:model/Table :name :schema] :id table-id {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))
(mu/defn table-database-id
"The ::warehouses.schema/database ID of the ::warehouse-schema.schema/table with `table-id`."
[table-id :- ::lib.schema.id/table]
- (t2/select-one-fn :db_id :model/Table table-id))
-
-(mu/defn update-table!
- "Apply `changes` to the ::warehouse-schema.schema/table with `table-id`, returning the number updated."
- [table-id :- ::lib.schema.id/table
- changes :- (mut/select-keys ::warehouse-schema.schema/table.update [:field_order])]
- (t2/update! :model/Table table-id changes))
+ (t2/select-one-fn :db_id :model/Table :id table-id {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))
(defn active-tables-for-database
"The active Tables of the Database with `database-id`."
[database-id]
- (t2/select :model/Table :db_id database-id :active true))
+ (t2/select :model/Table :db_id database-id :active true {:from [(warehouse-schema-overlay/table-query)]}))
(defn active-table-schemas
"The distinct schemas of the active Tables of the Database with `database-id`, in schema order. When
@@ -323,7 +457,8 @@
(let [clauses (cond-> []
(not include-hidden?) (conj [:= :visibility_type nil]))]
(t2/select-fn-set :schema :model/Table :db_id database-id :active true
- (merge {:order-by [[:%lower.schema :asc]]}
+ (merge {:from [(warehouse-schema-overlay/table-query)]
+ :order-by [[:%lower.schema :asc]]}
(when clauses
{:where (into [:and] clauses)})))))
@@ -334,7 +469,8 @@
:db_id database-id
:schema schema
:active true
- {:order-by [[:display_name :asc]]}))
+ {:from [(warehouse-schema-overlay/table-query)]
+ :order-by [[:display_name :asc]]}))
(defn active-visible-tables-in-schema
"The active, visible Tables in `schema` of the Database with `database-id`, in display name order."
@@ -344,7 +480,8 @@
:schema schema
:active true
:visibility_type nil
- {:order-by [[:display_name :asc]]}))
+ {:from [(warehouse-schema-overlay/table-query)]
+ :order-by [[:display_name :asc]]}))
(mu/defn unarchived-segments-for-tables
"The unarchived Segments of the Tables with `table-ids`, ordered by name."
@@ -378,6 +515,22 @@
[transform-ids :- [:sequential ::lib.schema.id/transform]]
(t2/select-fn->fn :id identity :model/Transform :id [:in transform-ids]))
+(mu/defn table-names-reducible
+ "A reducible of the id, name, and display name of every ::warehouse-schema.schema/table, plus the user's own
+ display name (`:user_display_name`), if any."
+ []
+ (t2/reducible-query
+ {:select [:t.id :t.name :t.display_name
+ [:u.display_name :user_display_name]]
+ :from [(warehouse-schema-overlay/table-query {:alias :t, :user-settings? false})]
+ :left-join [[(t2/table-name :model/TableUserSettings) :u] [:= :u.table_id :t.id]]}))
+
+(mu/defn set-table-display-name!
+ "Set the display name of the ::warehouse-schema.schema/table with `id`, returning the number updated."
+ [id :- ::lib.schema.id/table
+ display-name :- :string]
+ (t2/update! :model/Table id {:display_name display-name}))
+
;;; ------------------------------------------------ ::warehouses.schema/database ------------------------------------------------
(mu/defn database
diff --git a/src/metabase/warehouse_schema/humanization.clj b/src/metabase/warehouse_schema/humanization.clj
new file mode 100644
index 00000000000..2246bb759ff
--- /dev/null
+++ b/src/metabase/warehouse_schema/humanization.clj
@@ -0,0 +1,52 @@
+(ns metabase.warehouse-schema.humanization
+ "Humanization of table and field names, e.g. taking an identifier like `my_table` and returning a human-friendly
+ one like `My Table`. The Setting `humanization-strategy` determines which implementation is used: `:simple`, which
+ replaces underscores and dashes with spaces, or `:none`, an identity function that leaves names as-is."
+ (:require
+ [metabase.settings.core :as setting]
+ [metabase.util.humanization :as u.humanization]
+ [metabase.util.log :as log]
+ [metabase.util.malli :as mu]
+ [metabase.warehouse-schema.db :as warehouse-schema.db]))
+
+(defn name->human-readable-name
+ "Convert a name, such as `num_toucans`, to a human-readable name, such as `Num Toucans`. With one arg, this uses the
+ strategy defined by the Setting `humanization-strategy`. With two args, you may specify a custom strategy (intended
+ mainly for the internal implementation):
+
+ (humanization-strategy! :simple)
+ (name->human-readable-name \"cool_toucans\") ;-> \"Cool Toucans\"
+ ;; this is the same as:
+ (name->human-readable-name (humanization-strategy) \"cool_toucans\") ;-> \"Cool Toucans\"
+ ;; specify a different strategy:
+ (name->human-readable-name :none \"cool_toucans\") ;-> \"cool_toucans\""
+ ([s]
+ (name->human-readable-name (setting/get :humanization-strategy) s))
+ ([strategy s]
+ (u.humanization/name->human-readable-name strategy s)))
+
+(defn- re-humanize-names!
+ "Update the display name of every row of `entities-reducible` that the current strategy humanizes differently,
+ unless `custom?` of it."
+ [model entities-reducible set-display-name! custom?]
+ (run! (fn [{id :id, internal-name :name, display-name :display_name :as row}]
+ (let [new-strategy-display-name (name->human-readable-name internal-name)]
+ (when (and (not= display-name new-strategy-display-name)
+ (not (custom? row)))
+ (log/infof "Updating display name for %s '%s': '%s' -> '%s'"
+ (name model) internal-name display-name new-strategy-display-name)
+ (set-display-name! id new-strategy-display-name))))
+ (entities-reducible)))
+
+(mu/defn re-humanize-table-and-field-names!
+ "Re-humanize every Table and Field display name that is neither user-set nor different from `old-strategy`'s
+ humanization."
+ [old-strategy :- :keyword]
+ (letfn [(custom? [{internal-name :name, display-name :display_name
+ user-display-name :user_display_name}]
+ (or (some? user-display-name)
+ (not= (name->human-readable-name old-strategy internal-name) display-name)))]
+ (re-humanize-names! :model/Table warehouse-schema.db/table-names-reducible
+ warehouse-schema.db/set-table-display-name! custom?)
+ (re-humanize-names! :model/Field warehouse-schema.db/field-names-reducible
+ warehouse-schema.db/set-field-display-name! custom?)))
diff --git a/src/metabase/warehouse_schema/init.clj b/src/metabase/warehouse_schema/init.clj
new file mode 100644
index 00000000000..d4d5d9affc6
--- /dev/null
+++ b/src/metabase/warehouse_schema/init.clj
@@ -0,0 +1,3 @@
+(ns metabase.warehouse-schema.init
+ (:require
+ [metabase.warehouse-schema.settings]))
diff --git a/src/metabase/warehouse_schema/models/field.clj b/src/metabase/warehouse_schema/models/field.clj
index a811cbb31ca..a1b9f5c74d6 100644
--- a/src/metabase/warehouse_schema/models/field.clj
+++ b/src/metabase/warehouse_schema/models/field.clj
@@ -5,7 +5,6 @@
[metabase.app-db.core :as mdb]
[metabase.lib.core :as lib]
[metabase.lib.schema.metadata]
- [metabase.models.humanization :as humanization]
[metabase.models.interface :as mi]
[metabase.models.serialization :as serdes]
[metabase.permissions.core :as perms]
@@ -16,13 +15,14 @@
[metabase.util.log :as log]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[metabase.warehouse-schema.db :as warehouse-schema.db]
+ [metabase.warehouse-schema.humanization :as humanization]
[metabase.warehouse-schema.models.field-values :as field-values]
[metabase.warehouses.models.database :as database]
[methodical.core :as methodical]
[potemkin :as p]
[toucan2.core :as t2]
- [toucan2.protocols :as t2.protocols]
[toucan2.tools.hydrate :as t2.hydrate]))
(set! *warn-on-reflection* true)
@@ -106,7 +106,13 @@
{:in mi/json-in
:out (comp update-semantic-numeric-values mi/json-out-with-keywordization)})
-(t2/deftransforms :model/Field
+(def ^:private transform-field-boolean
+ "Boolean column transform; a boolean computed in SQL (e.g. `COALESCE` over `json_unfolding`) comes back as a number
+ from MySQL."
+ {:in identity
+ :out (fn [v] (if (number? v) (pos? v) v))})
+
+(def ^:private field-transforms
{:base_type transform-field-base-type
:effective_type transform-field-effective-type
:coercion_strategy transform-field-coercion-strategy
@@ -116,7 +122,10 @@
:data_sensitivity mi/transform-keyword
:fingerprint transform-json-fingerprints
:settings mi/transform-json
- :nfc_path mi/transform-json})
+ :nfc_path mi/transform-json
+ :json_unfolding transform-field-boolean})
+
+(t2/deftransforms :model/Field field-transforms)
(doto :model/Field
(derive :metabase/model)
@@ -163,32 +172,15 @@
enforce-effective-type-invariant)))
(def field-user-settings
- "Set of user-settable values for a Field"
- #{:semantic_type :description :display_name :visibility_type :has_field_values :effective_type :coercion_strategy :fk_target_field_id
- :caveats :points_of_interest :nfc_path :json_unfolding :settings :data_sensitivity})
-
-(defn- ensure-field-user-settings-exist-for-fk-target-field [field]
- (warehouse-schema.db/insert-field-user-settings!
- (map (fn [{:keys [id]}] {:field_id id})
- (warehouse-schema.db/fk-source-field-ids-without-user-settings (:id field)))))
-
-(defn- sync-user-settings [field]
- ;; we transparently prevent updates that would override user-set values
- (let [user-settings (warehouse-schema.db/field-user-settings (:id field))
- updated-field (-> (merge field (u/select-keys-when user-settings :non-nil field-user-settings))
- ;; GHY-3388 invariant: enforce coercion_strategy=nil ⇒ effective_type=base_type
- ;; AFTER the user-settings merge, since the overlay can introduce stale effective_type
- enforce-effective-type-invariant)]
- (t2.protocols/with-current field updated-field)))
+ "Set of user-settable values for a Field; see [[warehouse-schema-overlay/user-settable-field-columns]]."
+ warehouse-schema-overlay/user-settable-field-columns)
(t2/define-before-update :model/Field
[field]
(when (false? (:active (t2/changes field)))
- (ensure-field-user-settings-exist-for-fk-target-field field)
(warehouse-schema.db/clear-fk-targets-to-field! (:id field))
- ;; we must explicitly clear user-set fks in this case
(warehouse-schema.db/clear-user-settings-fk-targets-to-field! (:id field)))
- (sync-user-settings field))
+ (enforce-effective-type-invariant field))
(t2/define-before-delete :model/Field
[field]
@@ -476,13 +468,13 @@
#{[db-path]}))
(defmethod serdes/make-spec "Field" [_model-name opts]
- {:copy [:active :base_type :caveats :coercion_strategy :custom_position :data_sensitivity :database_default :database_indexed
+ {:copy [:active :base_type :caveats :coercion_strategy :data_sensitivity :database_default :database_indexed
:database_is_auto_increment :database_is_generated :database_is_nullable :database_is_pk
:database_partitioned :database_position :database_required :database_type
:description :display_name :effective_type :has_field_values :is_defective_duplicate
:json_unfolding :name :nfc_path :points_of_interest :position :preview_display :semantic_type :settings
:unique_field_helper :visibility_type]
- :skip [:dimension_interestingness :fingerprint :fingerprint_version :last_analyzed]
+ :skip [:custom_position :dimension_interestingness :fingerprint :fingerprint_version :last_analyzed]
:transform {:created_at (serdes/date)
:table_id (serdes/fk :model/Table)
:fk_target_field_id (serdes/fk :model/Field)
diff --git a/src/metabase/warehouse_schema/models/field_user_settings.clj b/src/metabase/warehouse_schema/models/field_user_settings.clj
index 8b5f6fa2582..60019d8fd6c 100644
--- a/src/metabase/warehouse_schema/models/field_user_settings.clj
+++ b/src/metabase/warehouse_schema/models/field_user_settings.clj
@@ -1,10 +1,14 @@
(ns metabase.warehouse-schema.models.field-user-settings
(:require
+ [metabase.lib.schema.id :as lib.schema.id]
[metabase.models.interface :as mi]
[metabase.models.serialization :as serdes]
[metabase.util :as u]
+ [metabase.util.malli :as mu]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[metabase.warehouse-schema.db :as warehouse-schema.db]
[metabase.warehouse-schema.models.field :as field]
+ [metabase.warehouse-schema.schema :as warehouse-schema.schema]
[methodical.core :as methodical]
[toucan2.core :as t2]))
@@ -26,15 +30,79 @@
(derive :metabase/model)
(derive :hook/timestamped?))
+(defn- with-set-flags
+ "Set the `_set` flag of every flagged column `effective` writes, unless `effective` sets the flag itself."
+ [settings effective]
+ (reduce-kv (fn [m column flag]
+ (cond-> m
+ (and (contains? effective column) (not (contains? effective flag)))
+ (assoc flag true)))
+ settings
+ warehouse-schema-overlay/field-user-settings-flags))
+
(methodical/defmethod t2/primary-keys :model/FieldUserSettings [_model] [:field_id])
-(defn upsert-user-settings
- "Upsert FieldUserSettings"
- [{:keys [id]} settings]
- (let [filtered-settings (u/select-keys-when settings :present field/field-user-settings)]
- (when-not (warehouse-schema.db/field-user-settings-exist? id)
- (warehouse-schema.db/insert-field-user-settings! {:field_id id}))
- (warehouse-schema.db/update-field-user-settings! id filtered-settings)))
+(defn- delete-when-empty!
+ "Delete `settings` when it holds no user value and no true flag, returning it either way."
+ [settings]
+ (when (and (every? #(nil? (get settings %)) warehouse-schema-overlay/user-settable-field-columns)
+ (not-any? #(get settings %) (vals warehouse-schema-overlay/field-user-settings-flags)))
+ (warehouse-schema.db/delete-field-user-settings! (:field_id settings)))
+ settings)
+
+(t2/define-before-insert :model/FieldUserSettings
+ [settings]
+ (with-set-flags settings settings))
+
+(t2/define-after-insert :model/FieldUserSettings
+ [settings]
+ (delete-when-empty! settings))
+
+(t2/define-before-update :model/FieldUserSettings
+ [settings]
+ (with-set-flags settings (t2/changes settings)))
+
+(t2/define-after-update :model/FieldUserSettings
+ [settings]
+ (delete-when-empty! settings))
+
+(mu/defn upsert-user-settings
+ "Record the user-settable Field columns present in `settings` as the user values of `field`."
+ [{:keys [id]} :- [:map [:id ::lib.schema.id/field]]
+ settings :- ::warehouse-schema.schema/field.update]
+ (let [settings (u/select-keys-when settings :present field/field-user-settings)]
+ (when (seq settings)
+ (if (warehouse-schema.db/field-user-settings-exist? id)
+ (warehouse-schema.db/update-field-user-settings! id (with-set-flags settings settings))
+ (warehouse-schema.db/insert-field-user-settings! (assoc settings :field_id id))))))
+
+(mu/defn set-custom-positions!
+ "Record `field-id->position` as the Fields' user `custom_position`s."
+ [field-id->position :- [:map-of ::lib.schema.id/field :int]]
+ (let [existing (warehouse-schema.db/field-ids-with-user-settings (keys field-id->position))
+ missing (remove existing (keys field-id->position))]
+ (when (seq missing)
+ (warehouse-schema.db/insert-field-user-settings!
+ (mapv (fn [id] {:field_id id :custom_position (field-id->position id)}) missing)))
+ (when (seq existing)
+ (warehouse-schema.db/update-field-user-settings-custom-positions! (select-keys field-id->position existing)))))
+
+(mu/defn unset-user-settings!
+ "Drop the user values of the Field columns `ks` for `field`."
+ [{:keys [id]} :- [:map [:id ::lib.schema.id/field]]
+ ks :- [:sequential (into [:enum] warehouse-schema-overlay/user-settable-field-columns)]]
+ (when (warehouse-schema.db/field-user-settings-exist? id)
+ (warehouse-schema.db/update-field-user-settings!
+ id
+ (into {} (mapcat (fn [k]
+ (cond-> [[k nil]]
+ (warehouse-schema-overlay/field-user-settings-flags k) (conj [(warehouse-schema-overlay/field-user-settings-flags k) false]))))
+ ks))))
+
+(defmethod serdes/extract-query "FieldUserSettings" [_model-name {:keys [filter-column filter-ids] :as opts}]
+ (if (= filter-column :table_id)
+ (warehouse-schema.db/field-user-settings-for-tables filter-ids)
+ (serdes/extract-query-collections :model/FieldUserSettings opts)))
(defmethod serdes/entity-id "FieldUserSettings" [_ _] nil)
@@ -42,15 +110,6 @@
(conj (serdes/generate-path "Field" {:id field_id})
{:model "FieldUserSettings" :id "1"}))
-(defmethod serdes/deserialization-dependencies "FieldUserSettings" [fv]
- (let [db-path (first (serdes/path fv))]
- [[db-path]]))
-
-(defmethod serdes/load-find-local "FieldUserSettings" [path]
- ;; Delegate to finding the parent Field, then look up its corresponding FieldUserSettings.
- (let [field (serdes/load-find-local (pop path))]
- (warehouse-schema.db/field-user-settings (:id field))))
-
(defn- field-path->field-ref [field-values-path]
(let [[db schema table field :as field-ref] (map :id (pop field-values-path))]
(if field
@@ -61,7 +120,11 @@
(defmethod serdes/make-spec "FieldUserSettings" [_model-name _opts]
{:copy [:semantic_type :description :display_name :visibility_type
:has_field_values :effective_type :coercion_strategy :caveats
- :points_of_interest :nfc_path :json_unfolding :settings :data_sensitivity]
+ :points_of_interest :nfc_path :json_unfolding :settings :data_sensitivity :custom_position
+ :description_set :semantic_type_set :fk_target_field_id_set]
+ :defaults {:description_set false
+ :semantic_type_set false
+ :fk_target_field_id_set false}
:transform {:created_at (serdes/date)
:fk_target_field_id (serdes/fk :model/Field)
:field_id {::serdes/fk true
@@ -69,13 +132,3 @@
:import-with-context (fn [current _ _]
(let [field-ref (field-path->field-ref (serdes/path current))]
(serdes/*import-field-fk* field-ref)))}}})
-
-(def ^:private field-values-slug "___fieldusersettings")
-
-(defmethod serdes/storage-path "FieldUserSettings" [fv _]
- ;; [path to table "fields" "field-name___fieldusersettings"] since there's zero or one FieldUserSettings per Field, and Fields
- ;; don't have their own directories.
- (let [hierarchy (serdes/path fv)
- field-path (serdes/storage-path-prefixes (drop-last hierarchy))]
- (update field-path (dec (count field-path))
- (fn [segment] (update segment :label str field-values-slug)))))
diff --git a/src/metabase/warehouse_schema/models/table.clj b/src/metabase/warehouse_schema/models/table.clj
index 06d5ca76496..1390222863a 100644
--- a/src/metabase/warehouse_schema/models/table.clj
+++ b/src/metabase/warehouse_schema/models/table.clj
@@ -1,10 +1,10 @@
(ns metabase.warehouse-schema.models.table
(:require
+ [medley.core :as m]
[metabase.api.common :as api]
[metabase.audit-app.core :as audit]
[metabase.collections.models.collection :as collection]
[metabase.driver :as driver]
- [metabase.models.humanization :as humanization]
[metabase.models.interface :as mi]
[metabase.models.serialization :as serdes]
[metabase.permissions.core :as perms]
@@ -14,7 +14,9 @@
[metabase.util :as u]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[metabase.warehouse-schema.db :as warehouse-schema.db]
+ [metabase.warehouse-schema.humanization :as humanization]
[methodical.core :as methodical]
[toucan2.core :as t2]))
@@ -38,7 +40,7 @@
:hidden - low quality, hidden, not synced"
#{:final :internal :hidden})
-(defn- visibility-type->data-layer
+(defn visibility-type->data-layer
"Convert legacy visibility_type to data_layer.
Used when updating via the legacy field."
[visibility-type]
@@ -46,7 +48,7 @@
:hidden
:internal))
-(defn- data-layer->visibility-type
+(defn data-layer->visibility-type
"Convert data_layer back to legacy visibility_type.
Used for rollback compatibility to v56."
[data-layer]
@@ -95,23 +97,27 @@
;; cause duplication rather than good matching if the two instances are later linked by serdes.
#_(derive :hook/entity-id))
-(def ^:private transform-data-authority
+(def transform-data-authority
+ "Transform for a `data_authority` column. NULL passes through: the column is NOT NULL on `metabase_table`, but
+ nullable in `metabase_table_user_settings`, where NULL means the user set nothing."
{:out (fn [value]
- (let [kw (some-> value keyword)]
- (if (contains? writable-data-authority-types kw)
- kw
- (do (log/warnf "Unknown data_authority value from database: %s, converting to :unknown" value)
- :unknown))))
+ (when (some? value)
+ (let [kw (keyword value)]
+ (if (contains? writable-data-authority-types kw)
+ kw
+ (do (log/warnf "Unknown data_authority value from database: %s, converting to :unknown" value)
+ :unknown)))))
:in (fn [value]
- (let [kw (some-> value keyword)]
- (when-not (contains? writable-data-authority-types kw)
- (throw (ex-info (str "Illegal value for data_authority: " kw)
- {:field :data_authority
- :value value
- :status-code 400}))))
- (some-> value name))})
-
-(def ^:private legacy-data-layer->current
+ (when (some? value)
+ (let [kw (keyword value)]
+ (when-not (contains? writable-data-authority-types kw)
+ (throw (ex-info (str "Illegal value for data_authority: " kw)
+ {:field :data_authority
+ :value value
+ :status-code 400})))
+ (name kw))))})
+
+(def legacy-data-layer->current
"Map old medallion data_layer values to current values.
Used to handle values from pre-v59 databases or serialization exports."
{:copper :hidden
@@ -119,29 +125,81 @@
:silver :final
:gold :final})
+(def ^:private transform-table-boolean
+ "Boolean column transform; a boolean computed in SQL (the merge in `table-query`) comes back as a number from MySQL
+ and MariaDB, which have no boolean type of their own."
+ {:in identity
+ :out (fn [v] (if (number? v) (pos? v) v))})
+
(t2/deftransforms :model/Table
- {:entity_type mi/transform-keyword
- :visibility_type mi/transform-keyword
- :data_layer (mi/transform-validator-with-fixes
- mi/transform-keyword
- (partial mi/assert-optional-enum data-layers)
- (some-fn legacy-data-layer->current identity))
- :field_order mi/transform-keyword
- :data_source (mi/transform-validator-with-fixes
- mi/transform-keyword
- (partial mi/assert-optional-enum data-sources)
- (some-fn keyword identity))
+ {:entity_type mi/transform-keyword
+ :is_published transform-table-boolean
+ :show_in_getting_started transform-table-boolean
+ :visibility_type mi/transform-keyword
+ :data_layer (mi/transform-validator-with-fixes
+ mi/transform-keyword
+ (partial mi/assert-optional-enum data-layers)
+ (some-fn legacy-data-layer->current identity))
+ :field_order mi/transform-keyword
+ :data_source (mi/transform-validator-with-fixes
+ mi/transform-keyword
+ (partial mi/assert-optional-enum data-sources)
+ (some-fn keyword identity))
;; Warning: by using a transform to handle unexpected enum values, serialization becomes lossy
- :data_authority transform-data-authority})
-
-(methodical/defmethod t2/model-for-automagic-hydration [:default :table]
- [_original-model _k]
- :model/Table)
+ :data_authority transform-data-authority})
+
+(mi/define-batched-hydration-method with-table
+ :table
+ "Hydrate the Table of each of `instances` from its `table_id`.
+
+ Batched rather than automagic because automagic hydration selects the model's own row, which shows what sync wrote:
+ `mi/can-read?`/`can-query?` read `is_published` and `collection_id` off the hydrated Table to decide access through
+ a published collection, and those are user values."
+ [instances]
+ (let [table-ids (into #{} (keep :table_id) instances)
+ id->table (when (seq table-ids)
+ (m/index-by :id (warehouse-schema.db/tables table-ids)))]
+ (for [instance instances]
+ (m/assoc-some instance :table (get id->table (:table_id instance))))))
(t2/define-after-select :model/Table
[table]
(dissoc table :is_defective_duplicate :unique_table_helper))
+(defn validate-user-changes!
+ "Throw a 400 for a change in `changes` a user is not allowed to make to `original-table`."
+ [changes original-table]
+ ;; Don't allow tables to be moved into collections which are not part of the Library's "Data" collection.
+ ;; Tables can be moved out of any collection, however.
+ (when (:collection_id changes)
+ (collection/check-allowed-content :table (:collection_id changes)))
+ ;; Prevent setting data_authority back to unconfigured once configured
+ (when (and (not= (keyword (:data_authority original-table :unconfigured)) :unconfigured)
+ (= (keyword (:data_authority changes)) :unconfigured))
+ (throw (ex-info "Cannot set data_authority back to unconfigured once it has been configured"
+ {:status-code 400})))
+ ;; Prevent changing data_source to/from metabase-transform.
+ ;; The "to metabase-transform" direction is allowed during deserialization so an existing synced table
+ ;; can be migrated to a transform-managed table via serdes.
+ (when (contains? changes :data_source)
+ (let [original-data-source (some-> (:data_source original-table) keyword)
+ new-data-source (some-> (:data_source changes) keyword)]
+ (when (and (= original-data-source :metabase-transform)
+ (not= new-data-source :metabase-transform))
+ (throw (ex-info "Cannot change data_source from metabase-transform"
+ {:status-code 400})))
+ (when (and (not mi/*deserializing?*)
+ (not= original-data-source :metabase-transform)
+ (= new-data-source :metabase-transform))
+ (throw (ex-info "Cannot set data_source to metabase-transform"
+ {:status-code 400})))))
+ (when (and (contains? changes :visibility_type)
+ (contains? changes :data_layer)
+ (not= (keyword (:visibility_type changes)) (keyword (:visibility_type original-table)))
+ (not= (keyword (:data_layer changes)) (keyword (:data_layer original-table))))
+ (throw (ex-info "Cannot update both visibility_type and data_layer"
+ {:status-code 400}))))
+
(defn- sync-visibility-fields
"Sync visibility_type and data_layer fields, ensuring only one is updated at a time.
Returns updated changes map with both fields in sync for rollback compatibility."
@@ -194,30 +252,7 @@
original-table (t2/original table)
current-active (:active original-table)
new-active (:active changes)]
- ;; Don't allow tables to be moved into collections which are not part of the Library's "Data" collection.
- ;; Tables can be moved out of any collection, however.
- (when (:collection_id changes)
- (collection/check-allowed-content :table (:collection_id changes)))
- ;; Prevent setting data_authority back to unconfigured once configured
- (when (and (not= (keyword (:data_authority original-table :unconfigured)) :unconfigured)
- (= (keyword (:data_authority changes)) :unconfigured))
- (throw (ex-info "Cannot set data_authority back to unconfigured once it has been configured"
- {:status-code 400})))
- ;; Prevent changing data_source to/from metabase-transform.
- ;; The "to metabase-transform" direction is allowed during deserialization so an existing synced table
- ;; can be migrated to a transform-managed table via serdes.
- (when (contains? changes :data_source)
- (let [original-data-source (some-> (:data_source original-table) keyword)
- new-data-source (some-> (:data_source changes) keyword)]
- (when (and (= original-data-source :metabase-transform)
- (not= new-data-source :metabase-transform))
- (throw (ex-info "Cannot change data_source from metabase-transform"
- {:status-code 400})))
- (when (and (not mi/*deserializing?*)
- (not= original-data-source :metabase-transform)
- (= new-data-source :metabase-transform))
- (throw (ex-info "Cannot set data_source to metabase-transform"
- {:status-code 400})))))
+ (validate-user-changes! changes original-table)
;; Sync visibility_type and data_layer fields
(let [changes (sync-visibility-fields changes original-table)]
(cond
@@ -439,24 +474,6 @@
(u/the-id table)
(:field_order table)))))
-(defn- valid-field-order?
- "Field ordering is valid if all the fields from a given table are present and only from that table."
- [table field-ordering]
- (= (warehouse-schema.db/active-field-ids-for-table (u/the-id table))
- (set field-ordering)))
-
-(defn custom-order-fields!
- "Set field order to `field-order`."
- [table field-order]
- {:pre [(valid-field-order? table field-order)]}
- (t2/with-transaction [_]
- (warehouse-schema.db/update-table! (u/the-id table) {:field_order :custom})
- (dorun
- (map-indexed (fn [position field-id]
- (warehouse-schema.db/update-field! field-id {:position position
- :custom_position position}))
- field-order))))
-
;;; --------------------------------------------------- Hydration ----------------------------------------------------
(methodical/defmethod t2/batched-hydrate [:model/Table :field_values]
@@ -553,20 +570,17 @@
collection_id (conj [{:model "Collection" :id collection_id}])
transform_id (conj [{:model "Transform" :id transform_id}])))
-(defmethod serdes/descendants "Table" [_model-name id {:keys [skip-archived user-edits-only]}]
- (let [;; When user-edits-only, emit only fields that have user-authored metadata (FieldUserSettings rows).
- ;; On import the parent Field row is synthesized if missing, so FieldUserSettings is self-sufficient.
- ;; Otherwise emit all Field ids for a full serdes backup/restore.
- fields (if user-edits-only
- (into {} (for [fus-field-id (warehouse-schema.db/user-edited-field-ids-for-table id)]
- [["FieldUserSettings" fus-field-id] {"Table" id}]))
- (into {} (for [field-id (warehouse-schema.db/field-ids-for-table id)]
- [["Field" field-id] {"Table" id}])))
+(defmethod serdes/descendants "Table" [_model-name id {:keys [skip-archived]}]
+ (let [fields (into {} (for [field-id (warehouse-schema.db/field-ids-for-table id)]
+ [["Field" field-id] {"Table" id}]))
+ settings (when (or (warehouse-schema.db/table-user-settings-exist? id)
+ (warehouse-schema.db/field-user-settings-exist-for-table? id))
+ {["TableUserSettings" id] {"Table" id}})
segments (into {} (for [segment-id (warehouse-schema.db/segment-ids-for-table id skip-archived)]
[["Segment" segment-id] {"Table" id}]))
measures (into {} (for [measure-id (warehouse-schema.db/measure-ids-for-table id skip-archived)]
[["Measure" measure-id] {"Table" id}]))]
- (merge fields segments measures)))
+ (merge fields settings segments measures)))
(defmethod serdes/generate-path "Table" [_ table]
(let [db-name (warehouse-schema.db/database-name (:db_id table))]
@@ -613,6 +627,7 @@
(search.spec/define-spec "table"
{:model :model/Table
+ :source #(warehouse-schema-overlay/table-query {:alias :this})
:attrs {;; legacy search uses :active for this, but then has a rule to only ever show active tables
;; so we moved that to the where clause
:archived false
@@ -622,7 +637,8 @@
:database-id :db_id
:view-count true
:created-at true
- :updated-at true
+ :updated-at [:case [:> :settings.updated_at :this.updated_at] :settings.updated_at
+ :else :this.updated_at]
:is-published :is_published
:collection-type :collection.type
:collection-location :collection.location
@@ -646,9 +662,10 @@
[:= :this.collection_id nil]] "Our analytics"
:else nil]]}
:where [:and
- :active
- [:= :visibility_type nil]
+ :this.active
+ [:= :this.visibility_type nil]
[:= :db.router_database_id nil]
- [:not= :db_id [:inline audit/audit-db-id]]]
+ [:not= :this.db_id [:inline audit/audit-db-id]]]
:joins {:db [:model/Database [:= :db.id :this.db_id]]
+ :settings [:model/TableUserSettings [:= :settings.table_id :this.id]]
:collection [:model/Collection [:and [:= :this.is_published true] [:= :collection.id :this.collection_id]]]}})
diff --git a/src/metabase/warehouse_schema/models/table_user_settings.clj b/src/metabase/warehouse_schema/models/table_user_settings.clj
new file mode 100644
index 00000000000..5ff16161f53
--- /dev/null
+++ b/src/metabase/warehouse_schema/models/table_user_settings.clj
@@ -0,0 +1,217 @@
+(ns metabase.warehouse-schema.models.table-user-settings
+ (:require
+ [metabase.lib.schema.id :as lib.schema.id]
+ [metabase.models.interface :as mi]
+ [metabase.models.serialization :as serdes]
+ [metabase.util :as u]
+ [metabase.util.malli :as mu]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
+ [metabase.warehouse-schema.db :as warehouse-schema.db]
+ [metabase.warehouse-schema.models.field-user-settings :as field-user-settings]
+ [metabase.warehouse-schema.models.table :as table]
+ [metabase.warehouse-schema.schema :as warehouse-schema.schema]
+ [methodical.core :as methodical]
+ [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(methodical/defmethod t2/table-name :model/TableUserSettings [_model] :metabase_table_user_settings)
+
+(t2/deftransforms :model/TableUserSettings
+ {:entity_type mi/transform-keyword
+ :visibility_type mi/transform-keyword
+ :field_order mi/transform-keyword
+ :data_layer (mi/transform-validator-with-fixes
+ mi/transform-keyword
+ (partial mi/assert-optional-enum table/data-layers)
+ (some-fn table/legacy-data-layer->current identity))
+ :data_source (mi/transform-validator-with-fixes
+ mi/transform-keyword
+ (partial mi/assert-optional-enum table/data-sources)
+ (some-fn keyword identity))
+ :data_authority table/transform-data-authority})
+
+(doto :model/TableUserSettings
+ (derive :metabase/model)
+ (derive :hook/timestamped?)
+ (derive :hook/search-index))
+
+(methodical/defmethod t2/primary-keys :model/TableUserSettings [_model] [:table_id])
+
+(defn- complete-pairs
+ "Fill in the other half of the two-column choices in `settings`: `visibility_type`/`data_layer`, and
+ `is_published`/`collection_id`."
+ [settings table]
+ (let [changing? (fn [k] (and (contains? settings k)
+ (not= (some-> (get settings k) keyword)
+ (some-> (get table k) keyword))))]
+ (cond-> settings
+ (and (changing? :visibility_type) (not (contains? settings :data_layer)))
+ (assoc :data_layer (table/visibility-type->data-layer (some-> (:visibility_type settings) keyword)))
+
+ (and (changing? :data_layer) (not (contains? settings :visibility_type)))
+ (assoc :visibility_type (table/data-layer->visibility-type (some-> (:data_layer settings) keyword)))
+
+ (and (contains? settings :collection_id) (not (contains? settings :is_published)))
+ (assoc :is_published (boolean (:is_published table))))))
+
+(defn- with-set-flags
+ "Set the `_set` flag of every flagged column in `effective`, unless `explicit` sets the flag itself."
+ [settings effective explicit]
+ (reduce-kv (fn [m column flag]
+ (cond-> m
+ (and (contains? effective column) (not (contains? explicit flag)))
+ (assoc flag true)))
+ settings
+ warehouse-schema-overlay/table-user-settings-flags))
+
+(defn- enforce-invariants
+ "Validate `explicit`, complete its two-column pairs and set its flags, merged into `settings`."
+ [settings explicit]
+ (let [table (warehouse-schema.db/table (:table_id settings))
+ completed (complete-pairs explicit table)]
+ (table/validate-user-changes! explicit table)
+ (-> (merge settings completed)
+ (with-set-flags completed explicit))))
+
+(defn- delete-when-empty!
+ "Delete `settings` when it holds no user value and no true flag, returning it either way."
+ [settings]
+ (when (and (every? #(nil? (get settings %)) warehouse-schema-overlay/user-settable-table-columns)
+ (not-any? #(get settings %) (vals warehouse-schema-overlay/table-user-settings-flags)))
+ (warehouse-schema.db/delete-table-user-settings! (:table_id settings)))
+ settings)
+
+(t2/define-before-insert :model/TableUserSettings
+ [settings]
+ (enforce-invariants settings settings))
+
+(t2/define-after-insert :model/TableUserSettings
+ [settings]
+ (delete-when-empty! settings))
+
+(t2/define-before-update :model/TableUserSettings
+ [settings]
+ (enforce-invariants settings (t2/changes settings)))
+
+(t2/define-after-update :model/TableUserSettings
+ [settings]
+ (delete-when-empty! settings))
+
+(mu/defn upsert-user-settings
+ "Record the user-settable Table columns present in `settings` as the user values of `table`."
+ [{:keys [id]} :- [:map [:id ::lib.schema.id/table]]
+ settings :- ::warehouse-schema.schema/table.update]
+ (let [settings (u/select-keys-when settings :present warehouse-schema-overlay/user-settable-table-columns)]
+ (when (seq settings)
+ (if (warehouse-schema.db/table-user-settings-exist? id)
+ (warehouse-schema.db/update-table-user-settings! id (with-set-flags settings settings settings))
+ (warehouse-schema.db/insert-table-user-settings! (assoc settings :table_id id))))))
+
+(mu/defn upsert-user-settings-for-tables!
+ "[[upsert-user-settings]] for every Table in `table-ids`, in one statement per kind."
+ [table-ids :- [:set ::lib.schema.id/table]
+ settings :- ::warehouse-schema.schema/table.update]
+ (let [settings (u/select-keys-when settings :present warehouse-schema-overlay/user-settable-table-columns)]
+ (when (and (seq settings) (seq table-ids))
+ (let [existing (warehouse-schema.db/table-ids-with-user-settings table-ids)]
+ (when-let [missing (not-empty (remove existing table-ids))]
+ (warehouse-schema.db/insert-table-user-settings! (mapv #(assoc settings :table_id %) missing)))
+ (when (seq existing)
+ (warehouse-schema.db/update-table-user-settings-for-tables!
+ existing (with-set-flags settings settings settings)))))))
+
+(defn- valid-field-order?
+ "Field ordering is valid if all the fields from a given table are present and only from that table."
+ [table field-ordering]
+ (= (warehouse-schema.db/active-field-ids-for-table (u/the-id table))
+ (set field-ordering)))
+
+(defn custom-order-fields!
+ "Set field order to `field-order`."
+ [table field-order]
+ {:pre [(valid-field-order? table field-order)]}
+ (t2/with-transaction [_]
+ (upsert-user-settings table {:field_order :custom})
+ (field-user-settings/set-custom-positions! (zipmap field-order (range)))
+ (table/update-field-positions! (warehouse-schema.db/table (u/the-id table)))))
+
+(mu/defn unset-user-settings!
+ "Drop the user values of the Table columns `ks` for `table`."
+ [{:keys [id]} :- [:map [:id ::lib.schema.id/table]]
+ ks :- [:sequential (into [:enum] warehouse-schema-overlay/user-settable-table-columns)]]
+ (when (warehouse-schema.db/table-user-settings-exist? id)
+ (warehouse-schema.db/update-table-user-settings!
+ id
+ (into {} (mapcat (fn [k]
+ (cond-> [[k nil]]
+ (warehouse-schema-overlay/table-user-settings-flags k)
+ (conj [(warehouse-schema-overlay/table-user-settings-flags k) false]))))
+ ks))))
+
+;;; ------------------------------------------------- Serialization -------------------------------------------------
+
+(defmethod serdes/extract-query "TableUserSettings" [_model-name {:keys [filter-ids] :as opts}]
+ (serdes/extract-reducible-nested "TableUserSettings" (dissoc opts :filter-column :filter-ids)
+ (warehouse-schema.db/table-user-settings-with-field-settings filter-ids)))
+
+(defmethod serdes/entity-id "TableUserSettings" [_ _] nil)
+
+(defmethod serdes/generate-path "TableUserSettings" [_ {:keys [table_id]}]
+ (conj (serdes/table->path (serdes/*export-table-fk* table_id))
+ {:model "TableUserSettings" :id "1"}))
+
+(defmethod serdes/deserialization-dependencies "TableUserSettings" [tus]
+ (let [db-path (first (serdes/path tus))]
+ (cond-> [[db-path]]
+ (:collection_id tus) (conj [{:model "Collection" :id (:collection_id tus)}]))))
+
+(defmethod serdes/load-one! "TableUserSettings" [ingested maybe-local]
+ (let [settings (serdes/default-load-one! ingested maybe-local)]
+ (when (:field_order ingested)
+ (table/update-field-positions! (warehouse-schema.db/table (:table_id settings))))
+ settings))
+
+(defmethod serdes/load-find-local "TableUserSettings" [path]
+ (let [found-table (serdes/load-find-local (pop path))]
+ (warehouse-schema.db/table-user-settings (:id found-table))))
+
+(defn- table-path->table-ref [tus-path]
+ (let [[db schema table-name :as table-ref] (mapv :id (pop tus-path))]
+ (if table-name
+ table-ref
+ ;; It's too short, so no schema. Shift them over and add a nil schema.
+ [db nil schema])))
+
+(defmethod serdes/make-spec "TableUserSettings" [_model-name _opts]
+ {:copy [:display_name :description :entity_type :visibility_type :field_order :caveats :points_of_interest
+ :show_in_getting_started :data_authority :data_source :owner_email :is_published
+ :description_set :visibility_type_set :caveats_set
+ :points_of_interest_set :data_layer_set :data_source_set]
+ :defaults {:description_set false
+ :visibility_type_set false
+ :caveats_set false
+ :points_of_interest_set false
+ :data_layer_set false
+ :data_source_set false}
+ :transform {:created_at (serdes/date)
+ :collection_id (serdes/fk :model/Collection)
+ :owner_user_id (serdes/fk :model/User)
+ :data_layer (serdes/optional-kw)
+ :table_id {::serdes/fk true
+ :export (constantly ::serdes/skip)
+ :import-with-context (fn [current _ _]
+ (serdes/*import-table-fk* (table-path->table-ref (serdes/path current))))}
+ :fields (serdes/nested :model/FieldUserSettings :table_id
+ {:sort-by :field_name
+ :delete-children! warehouse-schema.db/delete-field-user-settings-for-table!})}})
+
+(def ^:private table-user-settings-slug "___tableusersettings")
+
+(defmethod serdes/storage-path "TableUserSettings" [tus {:keys [inline-user-settings]}]
+ (let [table-path (pop (serdes/path tus))]
+ (if inline-user-settings
+ (conj (serdes/storage-path-prefixes table-path)
+ {:label (:id (peek table-path)) :key (:id (peek table-path))})
+ (conj (serdes/storage-path-prefixes table-path)
+ {:label (str (:id (peek table-path)) table-user-settings-slug)}))))
diff --git a/src/metabase/warehouse_schema/schema.clj b/src/metabase/warehouse_schema/schema.clj
index 6c32cc1a45d..a036a249fdf 100644
--- a/src/metabase/warehouse_schema/schema.clj
+++ b/src/metabase/warehouse_schema/schema.clj
@@ -143,7 +143,11 @@
[:nfc_path [:maybe [:sequential [:or :string :keyword]]]]
[:json_unfolding [:maybe :boolean]]
[:settings [:maybe ::field-user-settings.settings]]
- [:data_sensitivity [:maybe [:or :keyword :string]]]])
+ [:data_sensitivity [:maybe [:or :keyword :string]]]
+ [:custom_position [:maybe :int]]
+ [:description_set :boolean]
+ [:semantic_type_set :boolean]
+ [:fk_target_field_id_set :boolean]])
(mr/def ::field-user-settings.update
"What an update (or insert) of a FieldUserSettings accepts: every column of `:metabase_field_user_settings` except `id`, all optional."
@@ -164,7 +168,11 @@
[:nfc_path {:optional true} [:maybe [:sequential [:or :string :keyword]]]]
[:json_unfolding {:optional true} [:maybe :boolean]]
[:settings {:optional true} [:maybe ::field-user-settings.settings]]
- [:data_sensitivity {:optional true} [:maybe [:or :keyword :string]]]])
+ [:data_sensitivity {:optional true} [:maybe [:or :keyword :string]]]
+ [:custom_position {:optional true} [:maybe :int]]
+ [:description_set {:optional true} :boolean]
+ [:semantic_type_set {:optional true} :boolean]
+ [:fk_target_field_id_set {:optional true} :boolean]])
(mr/def ::field-values
"A FieldValues as selected from the app DB: every column of `:metabase_fieldvalues`."
@@ -193,6 +201,63 @@
[:hash_key {:optional true} [:maybe :string]]
[:last_used_at {:optional true} [:maybe ms/TemporalInstant]]])
+(mr/def ::table-user-settings
+ "A TableUserSettings as selected from the app DB: every column of `:metabase_table_user_settings`."
+ [:map {:closed true}
+ [:table_id ::lib.schema.id/table]
+ [:created_at ms/TemporalInstant]
+ [:updated_at ms/TemporalInstant]
+ [:display_name [:maybe :string]]
+ [:description [:maybe :string]]
+ [:entity_type [:maybe [:or :keyword :string]]]
+ [:visibility_type [:maybe [:or :keyword :string]]]
+ [:caveats [:maybe :string]]
+ [:points_of_interest [:maybe :string]]
+ [:data_layer [:maybe [:or :keyword :string]]]
+ [:data_source [:maybe [:or :keyword :string]]]
+ [:owner_email [:maybe :string]]
+ [:owner_user_id [:maybe ::lib.schema.id/user]]
+ [:field_order [:maybe [:or :keyword :string]]]
+ [:show_in_getting_started [:maybe :boolean]]
+ [:data_authority [:maybe [:or :keyword :string]]]
+ [:is_published [:maybe :boolean]]
+ [:collection_id [:maybe ::lib.schema.id/collection]]
+ [:description_set :boolean]
+ [:visibility_type_set :boolean]
+ [:caveats_set :boolean]
+ [:points_of_interest_set :boolean]
+ [:data_layer_set :boolean]
+ [:data_source_set :boolean]])
+
+(mr/def ::table-user-settings.update
+ "What an update (or insert) of a TableUserSettings accepts: every column of
+ `:metabase_table_user_settings`, all optional."
+ [:map {:closed true}
+ [:table_id {:optional true} [:maybe ::lib.schema.id/table]]
+ [:created_at {:optional true} [:maybe ms/TemporalInstant]]
+ [:updated_at {:optional true} [:maybe ms/TemporalInstant]]
+ [:display_name {:optional true} [:maybe :string]]
+ [:description {:optional true} [:maybe :string]]
+ [:entity_type {:optional true} [:maybe [:or :keyword :string]]]
+ [:visibility_type {:optional true} [:maybe [:or :keyword :string]]]
+ [:caveats {:optional true} [:maybe :string]]
+ [:points_of_interest {:optional true} [:maybe :string]]
+ [:data_layer {:optional true} [:maybe [:or :keyword :string]]]
+ [:data_source {:optional true} [:maybe [:or :keyword :string]]]
+ [:owner_email {:optional true} [:maybe :string]]
+ [:owner_user_id {:optional true} [:maybe ::lib.schema.id/user]]
+ [:field_order {:optional true} [:maybe [:or :keyword :string]]]
+ [:show_in_getting_started {:optional true} [:maybe :boolean]]
+ [:data_authority {:optional true} [:maybe [:or :keyword :string]]]
+ [:is_published {:optional true} [:maybe :boolean]]
+ [:collection_id {:optional true} [:maybe ::lib.schema.id/collection]]
+ [:description_set {:optional true} :boolean]
+ [:visibility_type_set {:optional true} :boolean]
+ [:caveats_set {:optional true} :boolean]
+ [:points_of_interest_set {:optional true} :boolean]
+ [:data_layer_set {:optional true} :boolean]
+ [:data_source_set {:optional true} :boolean]])
+
(mr/def ::table
"A Table as selected from the app DB: every column of `:metabase_table`."
[:map {:closed true}
diff --git a/src/metabase/warehouse_schema/settings.clj b/src/metabase/warehouse_schema/settings.clj
new file mode 100644
index 00000000000..9fb9670592e
--- /dev/null
+++ b/src/metabase/warehouse_schema/settings.clj
@@ -0,0 +1,44 @@
+(ns metabase.warehouse-schema.settings
+ "Settings of the warehouse-schema module."
+ (:require
+ [metabase.settings.core :as setting :refer [defsetting]]
+ [metabase.util :as u]
+ [metabase.util.humanization :as u.humanization]
+ [metabase.util.i18n :refer [deferred-tru tru]]
+ [metabase.util.log :as log]
+ [metabase.warehouse-schema.humanization :as humanization]))
+
+(defn- set-humanization-strategy!
+ "Set `humanization-strategy` to `new-value`, throwing for an unknown strategy, then re-humanize every Table and
+ Field display name.
+
+ TODO: Should we do this in a background thread because it is potentially slow?
+ https://github.com/metabase/metabase/issues/39406"
+ [new-value]
+ (let [new-strategy (keyword (or new-value :simple))]
+ (when-not (get-method u.humanization/name->human-readable-name new-strategy)
+ (throw (IllegalArgumentException.
+ (tru "Invalid humanization strategy ''{0}''. Valid strategies are: {1}"
+ new-strategy (keys (methods u.humanization/name->human-readable-name))))))
+ (let [old-strategy (setting/get-value-of-type :keyword :humanization-strategy)]
+ (setting/set-value-of-type! :keyword :humanization-strategy new-value)
+ (log/infof "Changing Table & Field names humanization strategy from '%s' to '%s'"
+ (name old-strategy) (name new-strategy))
+ (humanization/re-humanize-table-and-field-names! old-strategy))))
+
+(defsetting ^{:added "0.28.0"} humanization-strategy
+ (deferred-tru
+ (str "To make table and field names more human-friendly, Metabase will replace dashes and underscores in them "
+ "with spaces. We’ll capitalize each word while at it, so ‘last_visited_at’ will become ‘Last Visited At’."))
+ :type :keyword
+ :default :simple
+ :visibility :settings-manager
+ :export? true
+ :audit :raw-value
+ :getter (fn []
+ (let [strategy (setting/get-value-of-type :keyword :humanization-strategy)
+ valid-values (set (keys (methods u.humanization/name->human-readable-name)))
+ valid-strategy? (contains? valid-values strategy)]
+ (when (not valid-strategy?) (log/warn (u/format-color :yellow "Invalid humanization strategy '%s'. Defaulting to 'simple'" strategy)))
+ (if valid-strategy? strategy :simple)))
+ :setter set-humanization-strategy!)
diff --git a/src/metabase/warehouse_schema_overlay/core.clj b/src/metabase/warehouse_schema_overlay/core.clj
new file mode 100644
index 00000000000..50eec856971
--- /dev/null
+++ b/src/metabase/warehouse_schema_overlay/core.clj
@@ -0,0 +1,167 @@
+(ns metabase.warehouse-schema-overlay.core
+ "Sources for queries over Fields and Tables: [[field-query]] and [[table-query]] merge each row with the values
+ in its user-settings table."
+ (:require
+ [metabase.util :as u]
+ [metabase.util.malli :as mu]
+ [toucan2.core :as t2]))
+
+(def user-settable-field-columns
+ "The Field columns users can set. Their user values live in `metabase_field_user_settings`, never in `metabase_field`."
+ #{:semantic_type :description :display_name :visibility_type :has_field_values :effective_type :coercion_strategy
+ :fk_target_field_id :caveats :points_of_interest :nfc_path :json_unfolding :settings :data_sensitivity
+ :custom_position})
+
+(def field-user-settings-flags
+ "The user-settable Field columns sync also writes, mapped to the flag that says the user set the column, NULL
+ included."
+ {:description :description_set
+ :semantic_type :semantic_type_set
+ :fk_target_field_id :fk_target_field_id_set})
+
+(def field-columns
+ "Every column of `metabase_field`. Spelled out rather than read from `:metabase.warehouse-schema.schema/field`,
+ which lives in a module above this one; `metabase.warehouse-schema-overlay.core-test` fails if the two drift."
+ #{:active :base_type :caveats :coercion_strategy :created_at :custom_position :data_sensitivity :database_default
+ :database_indexed :database_is_auto_increment :database_is_generated :database_is_nullable :database_is_pk
+ :database_partitioned :database_position :database_required :database_type :description
+ :dimension_interestingness :display_name :effective_type :fingerprint :fingerprint_version :fk_target_field_id
+ :has_field_values :id :is_defective_duplicate :json_unfolding :last_analyzed :name :nfc_path :parent_id
+ :points_of_interest :position :preview_display :semantic_type :settings :table_id :unique_field_helper
+ :updated_at :visibility_type})
+
+(def ^:private sync-owned-field-columns
+ "The columns of `metabase_field` users cannot set."
+ (sort (remove user-settable-field-columns field-columns)))
+
+(mu/defn- field-user-settings-join
+ "The `:left-join` entries joining `metabase_field_user_settings` as `settings-alias` to the Field table aliased
+ `field-alias`; see [[field-user-settings-column]]."
+ [field-alias :- :keyword
+ settings-alias :- :keyword]
+ [[(t2/table-name :model/FieldUserSettings) settings-alias]
+ [:= (u/qualified-key settings-alias :field_id) (u/qualified-key field-alias :id)]])
+
+(mu/defn- field-user-set-condition
+ "Honey SQL test for whether the user set `column`, or nil when a non-NULL value says so itself."
+ [column :- (into [:enum] user-settable-field-columns)
+ settings-alias :- :keyword]
+ (if-let [flag (field-user-settings-flags column)]
+ [:= (u/qualified-key settings-alias flag) true]
+ (when (= column :coercion_strategy)
+ [:not= (u/qualified-key settings-alias :effective_type) nil])))
+
+(mu/defn- field-user-settings-column
+ "Honey SQL expression for `column` as users see it: the user value when set, else the Field's."
+ [column :- (into [:enum] user-settable-field-columns)
+ field-alias :- :keyword
+ settings-alias :- :keyword]
+ (let [field-column (u/qualified-key field-alias column)
+ settings-column (u/qualified-key settings-alias column)]
+ (if-let [condition (field-user-set-condition column settings-alias)]
+ [:case condition settings-column :else field-column]
+ (if (= column :json_unfolding)
+ [:case [:= [:coalesce settings-column field-column] true] true :else false]
+ [:coalesce settings-column field-column]))))
+
+(mu/defn field-query :- [:tuple :any :keyword]
+ "The source a query over Fields reads from: `metabase_field` merged with the user values. `:alias` names it for
+ joins; `{:user-settings? false}` gives sync's own values."
+ ([]
+ (field-query nil))
+
+ ([{:keys [alias user-settings?]
+ :or {alias (t2/table-name :model/Field)
+ user-settings? true}} :- [:maybe [:map
+ [:alias {:optional true} :keyword]
+ [:user-settings? {:optional true} :boolean]]]]
+ [(if user-settings?
+ ^:allow-subquery
+ {:select (into (mapv #(u/qualified-key :f %) sync-owned-field-columns)
+ (map (fn [column] [(field-user-settings-column column :f :u) column]))
+ (sort user-settable-field-columns))
+ :from [[(t2/table-name :model/Field) :f]]
+ :left-join (field-user-settings-join :f :u)}
+ (t2/table-name :model/Field))
+ alias]))
+
+(def user-settable-table-columns
+ "The Table columns users can set. Their user values live in `metabase_table_user_settings`, never in
+ `metabase_table`."
+ #{:display_name :description :entity_type :visibility_type :caveats :points_of_interest :data_layer :data_source
+ :owner_email :owner_user_id :field_order :show_in_getting_started :data_authority :is_published :collection_id})
+
+(def table-user-settings-flags
+ "The user-settable Table columns a user can clear, mapped to the flag that says the user set the column, NULL
+ included."
+ {:description :description_set
+ :visibility_type :visibility_type_set
+ :caveats :caveats_set
+ :points_of_interest :points_of_interest_set
+ :data_layer :data_layer_set
+ :data_source :data_source_set})
+
+(def table-columns
+ "Every column of `metabase_table`. Spelled out rather than read from `:metabase.warehouse-schema.schema/table`,
+ which lives in a module above this one; `metabase.warehouse-schema-overlay.core-test` fails if the two drift."
+ #{:active :archived_at :caveats :collection_id :created_at :data_authority :data_layer :data_source
+ :database_require_filter :db_id :deactivated_at :description :display_name :entity_type :estimated_row_count
+ :field_order :id :initial_sync_status :is_defective_duplicate :is_published :is_upload :is_writable :name
+ :owner_email :owner_user_id :points_of_interest :schema :show_in_getting_started :transform_id :transform_target
+ :unique_table_helper :updated_at :view_count :visibility_type})
+
+(def ^:private sync-owned-table-columns
+ "The columns of `metabase_table` users cannot set."
+ (sort (remove user-settable-table-columns table-columns)))
+
+(mu/defn- table-user-settings-join
+ "The `:left-join` entries joining `metabase_table_user_settings` as `settings-alias` to the Table table aliased
+ `table-alias`; see [[table-user-settings-column]]."
+ [table-alias :- :keyword
+ settings-alias :- :keyword]
+ [[(t2/table-name :model/TableUserSettings) settings-alias]
+ [:= (u/qualified-key settings-alias :table_id) (u/qualified-key table-alias :id)]])
+
+(mu/defn- table-user-set-condition
+ "Honey SQL test for whether the user set `column`, or nil when a non-NULL value says so itself; `collection_id`
+ follows `is_published`."
+ [column :- (into [:enum] user-settable-table-columns)
+ settings-alias :- :keyword]
+ (if-let [flag (table-user-settings-flags column)]
+ [:= (u/qualified-key settings-alias flag) true]
+ (when (= column :collection_id)
+ [:not= (u/qualified-key settings-alias :is_published) nil])))
+
+(mu/defn- table-user-settings-column
+ "Honey SQL expression for `column` as users see it: the user value when set, else the Table's."
+ [column :- (into [:enum] user-settable-table-columns)
+ table-alias :- :keyword
+ settings-alias :- :keyword]
+ (let [table-column (u/qualified-key table-alias column)
+ settings-column (u/qualified-key settings-alias column)]
+ (if-let [condition (table-user-set-condition column settings-alias)]
+ [:case condition settings-column :else table-column]
+ (if (#{:show_in_getting_started :is_published} column)
+ [:case [:= [:coalesce settings-column table-column] true] true :else false]
+ [:coalesce settings-column table-column]))))
+
+(mu/defn table-query :- [:tuple :any :keyword]
+ "The source a query over Tables reads from: `metabase_table` merged with the user values, with the options of
+ [[field-query]]."
+ ([]
+ (table-query nil))
+
+ ([{:keys [alias user-settings?]
+ :or {alias (t2/table-name :model/Table)
+ user-settings? true}} :- [:maybe [:map
+ [:alias {:optional true} :keyword]
+ [:user-settings? {:optional true} :boolean]]]]
+ [(if user-settings?
+ ^:allow-subquery
+ {:select (into (mapv #(u/qualified-key :t %) sync-owned-table-columns)
+ (map (fn [column] [(table-user-settings-column column :t :u) column]))
+ (sort user-settable-table-columns))
+ :from [[(t2/table-name :model/Table) :t]]
+ :left-join (table-user-settings-join :t :u)}
+ (t2/table-name :model/Table))
+ alias]))
diff --git a/src/metabase/warehouse_schema_rest/api/field.clj b/src/metabase/warehouse_schema_rest/api/field.clj
index 88cbcd6a58d..19927cf4341 100644
--- a/src/metabase/warehouse_schema_rest/api/field.clj
+++ b/src/metabase/warehouse_schema_rest/api/field.clj
@@ -160,7 +160,9 @@
[:settings {:optional true} [:maybe ms/VisualizationSettings]]
[:nfc_path {:optional true} [:maybe [:sequential ms/NonBlankString]]]
[:json_unfolding {:optional true} [:maybe :boolean]]]]
- (let [field (t2/hydrate (api/write-check :model/Field id) :dimensions)
+ (let [field (-> (warehouse-schema-rest.db/field id)
+ api/write-check
+ (t2/hydrate :dimensions))
new-semantic-type (keyword (get body :semantic_type (:semantic_type field)))
[effective-type coercion-strategy]
(cond (not (contains? body :coercion_strategy))
@@ -187,24 +189,18 @@
(not removed-fk?)
(not= (:display_name field) display-name))
(warehouse-schema-rest.db/rename-dimension-for-field! id display-name))
- ;; everything checks out, now update the field
- (api/check-500
- (t2/with-transaction [_conn]
- (when removed-fk?
- (clear-dimension-on-fk-change! field))
- (clear-dimension-on-type-change! field (:base_type field) new-semantic-type)
- (let [body (assoc body
- :fk_target_field_id (when-not removed-fk? fk-target-field-id)
- :effective_type effective-type
- :coercion_strategy coercion-strategy)]
- (schema.field-user-settings/upsert-user-settings field body)
- (warehouse-schema-rest.db/update-field!
- id
- (u/select-keys-when body
- {:present #{:caveats :description :fk_target_field_id :points_of_interest :semantic_type
- :coercion_strategy :effective_type :has_field_values :nfc_path :json_unfolding
- :data_sensitivity}
- :non-nil #{:display_name :visibility_type :settings}})))))
+ (t2/with-transaction [_conn]
+ (when removed-fk?
+ (clear-dimension-on-fk-change! field))
+ (clear-dimension-on-type-change! field (:base_type field) new-semantic-type)
+ (schema.field-user-settings/upsert-user-settings
+ field
+ (cond-> body
+ (or removed-fk? (contains? body :fk_target_field_id))
+ (assoc :fk_target_field_id (when-not removed-fk? fk-target-field-id))
+
+ (contains? body :coercion_strategy)
+ (assoc :effective_type effective-type :coercion_strategy coercion-strategy))))
(when (some? json-unfolding)
(update-nested-fields-on-json-unfolding-change! field json-unfolding))
;; return updated field. note the fingerprint on this might be out of date if the task below would replace them
@@ -318,7 +314,7 @@
_query-params
{value-pairs :values} :- [:map {:closed true}
[:values ms/FieldValuesList]]]
- (let [field (api/write-check :model/Field id)]
+ (let [field (api/write-check (warehouse-schema-rest.db/field id))]
(api/check (field-values/field-should-have-field-values? field)
[400 (str "You can only update the human readable values of a mapped values of a Field whose value of "
"`has_field_values` is `list` or whose 'base_type' is 'type/Boolean'.")])
@@ -404,8 +400,8 @@
[:remapped-id ms/PositiveInt]]
{:keys [value]} :- [:map {:closed true}
[:value ms/NonBlankString]]]
- (let [field (api/read-check :model/Field id)
- remapped-field (api/read-check :model/Field remapped-id)
+ (let [field (-> (warehouse-schema-rest.db/field id) api/check-404 api/read-check)
+ remapped-field (-> (warehouse-schema-rest.db/field remapped-id) api/check-404 api/read-check)
value (parameters.field/parse-query-param-value-for-field field value)]
(parameters.field/remapped-value field remapped-field value)))
diff --git a/src/metabase/warehouse_schema_rest/api/table.clj b/src/metabase/warehouse_schema_rest/api/table.clj
index a5e5edc9dcf..bdca117e839 100644
--- a/src/metabase/warehouse_schema_rest/api/table.clj
+++ b/src/metabase/warehouse_schema_rest/api/table.clj
@@ -33,6 +33,7 @@
[metabase.util.quick-task :as quick-task]
[metabase.warehouse-schema-rest.db :as warehouse-schema-rest.db]
[metabase.warehouse-schema.models.table :as table]
+ [metabase.warehouse-schema.models.table-user-settings :as schema.table-user-settings]
[metabase.warehouse-schema.table :as schema.table]
[metabase.xrays.core :as xrays]
[steffan-westcott.clj-otel.api.trace.span :as span]
@@ -139,7 +140,8 @@
(let [api-perm-check-fn (if include_editable_data_model
api/write-check
api/read-check)]
- (-> (api-perm-check-fn :model/Table id)
+ (-> (api/check-404 (warehouse-schema-rest.db/table id))
+ api-perm-check-fn
(t2/hydrate :db :pk_field :collection)
schema.table/present-table)))
@@ -188,7 +190,7 @@
(u/update-some :data_layer keyword)
(u/update-some :data_source keyword)
not-empty)]
- (warehouse-schema-rest.db/update-table! id changes))
+ (schema.table-user-settings/upsert-user-settings existing-table changes))
(let [updated-table (warehouse-schema-rest.db/table id)
changed-field-order? (not= (:field_order updated-table) (:field_order existing-table))]
(if changed-field-order?
@@ -435,7 +437,9 @@
[:sequential ms/PositiveInt]
[:map {:closed true} [:field_order [:sequential ms/PositiveInt]]]]]
(let [field-order (if (map? body) (:field_order body) body)]
- (-> (warehouse-schema-rest.db/table id) api/write-check (table/custom-order-fields! field-order)))
+ (-> (warehouse-schema-rest.db/table id) api/write-check (schema.table-user-settings/custom-order-fields! field-order))
+ (events/publish-event! :event/table-update {:object (warehouse-schema-rest.db/table id)
+ :user-id api/*current-user-id*}))
{:success true})
(mu/defn- update-csv!
diff --git a/src/metabase/warehouse_schema_rest/db.clj b/src/metabase/warehouse_schema_rest/db.clj
index 560f7979204..c01ce648929 100644
--- a/src/metabase/warehouse_schema_rest/db.clj
+++ b/src/metabase/warehouse_schema_rest/db.clj
@@ -9,6 +9,7 @@
[metabase.util.honey-sql-2 :as h2x]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[metabase.warehouse-schema.schema :as warehouse-schema.schema]
[toucan2.core :as t2]))
@@ -18,22 +19,16 @@
target-field-id :- ::lib.schema.id/field]
(t2/query {:select [[:source_t.db_id :source_db_id]
[:target_t.db_id :target_db_id]]
- :from [[(t2/table-name :model/Field) :sf]]
- :join [[(t2/table-name :model/Table) :source_t] [:= :sf.table_id :source_t.id]
- [(t2/table-name :model/Field) :tf] [:= :tf.id target-field-id]
- [(t2/table-name :model/Table) :target_t] [:= :tf.table_id :target_t.id]]
+ :from [(warehouse-schema-overlay/field-query {:alias :sf})]
+ :join [(warehouse-schema-overlay/table-query {:alias :source_t}) [:= :sf.table_id :source_t.id]
+ (warehouse-schema-overlay/field-query {:alias :tf}) [:= :tf.id target-field-id]
+ (warehouse-schema-overlay/table-query {:alias :target_t}) [:= :tf.table_id :target_t.id]]
:where [:= :sf.id source-field-id]}))
(mu/defn field
"The Field with `field-id`, or nil."
[field-id :- ::lib.schema.id/field]
- (t2/select-one :model/Field :id field-id))
-
-(mu/defn update-field!
- "Apply `changes` to the Field with `field-id`."
- [field-id :- ::lib.schema.id/field
- changes :- ::warehouse-schema.schema/field.update]
- (t2/update! :model/Field field-id changes))
+ (t2/select-one :model/Field :id field-id {:from [(warehouse-schema-overlay/field-query)]}))
(mu/defn set-nested-fields-active!
"Set the active flag of the Fields of the Table with `table-id` whose NFC path matches the SQL LIKE
@@ -128,23 +123,17 @@
:where [:and
[:= :d.to_entity_id :metabase_table.id]
[:= :d.to_entity_type "table"]]}]))]
- (t2/select :model/Table {:where where, :order-by [[:name :asc]]})))
+ (t2/select :model/Table {:from [(warehouse-schema-overlay/table-query)], :where where, :order-by [[:name :asc]]})))
(mu/defn tables-by-ids
"The Tables with `table-ids`."
[table-ids :- [:sequential ::lib.schema.id/table]]
- (t2/select :model/Table :id [:in table-ids]))
+ (t2/select :model/Table :id [:in table-ids] {:from [(warehouse-schema-overlay/table-query)]}))
(mu/defn table
"The Table with `table-id`, or nil."
[table-id :- [:maybe ::lib.schema.id/table]]
- (t2/select-one :model/Table :id table-id))
-
-(mu/defn update-table!
- "Apply `changes` to the Table with `table-id`."
- [table-id :- ::lib.schema.id/table
- changes :- ::warehouse-schema.schema/table.update]
- (t2/update! :model/Table table-id changes))
+ (t2/select-one :model/Table :id table-id {:from [(warehouse-schema-overlay/table-query)]}))
(mu/defn database
"The Database with `database-id`, or nil."
@@ -164,17 +153,18 @@
(mu/defn active-unretired-field-ids-for-table
"The ids of the active, unretired Fields of the Table with `table-id`, or nil."
[table-id :- ::lib.schema.id/table]
- (t2/select-pks-set :model/Field, :table_id table-id, :visibility_type [:not= "retired"], :active true))
+ (t2/select-pks-set :model/Field :table_id table-id :visibility_type [:not= "retired"] :active true
+ {:from [(warehouse-schema-overlay/field-query)]}))
(mu/defn field-ids-for-table
"The ids of the Fields of the Table with `table-id`, or nil."
[table-id :- ::lib.schema.id/table]
- (t2/select-pks-set :model/Field :table_id table-id))
+ (t2/select-pks-set :model/Field :table_id table-id {:from [(warehouse-schema-overlay/field-query {:user-settings? false})]}))
(mu/defn active-fields-targeting
- "The active Fields whose FK target is one of `field-ids`."
+ "The active Fields whose FK target, as users see it, is one of `field-ids`."
[field-ids :- [:sequential ::lib.schema.id/field]]
- (t2/select :model/Field, :fk_target_field_id [:in field-ids], :active true))
+ (t2/select :model/Field :fk_target_field_id [:in field-ids] :active true {:from [(warehouse-schema-overlay/field-query)]}))
(mu/defn delete-field-values-for-fields!
"Delete the FieldValues of the Fields with `field-ids`."
diff --git a/src/metabase/warehouses/db.clj b/src/metabase/warehouses/db.clj
index b75faa3026a..e85bb02d66c 100644
--- a/src/metabase/warehouses/db.clj
+++ b/src/metabase/warehouses/db.clj
@@ -5,6 +5,7 @@
[metabase.app-db.core :as mdb]
[metabase.lib.schema.id :as lib.schema.id]
[metabase.util.malli :as mu]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[metabase.warehouses.schema :as warehouses.schema]
[toucan2.core :as t2]))
@@ -63,7 +64,7 @@
(mu/defn fields-exist-for-database?
"Whether any Field belongs to a Table of the Database with `database-id`."
[database-id :- ::lib.schema.id/database]
- (t2/exists? :model/Field :table_id [:in (table-ids-of-database-query database-id)]))
+ (t2/exists? :model/Field :table_id [:in (table-ids-of-database-query database-id)] {:from [(warehouse-schema-overlay/field-query)]}))
(mu/defn delete-childless-fields-for-database!
"Delete the childless Fields of the Tables of the Database with `database-id`, returning the number deleted."
@@ -115,7 +116,8 @@
(mu/defn active-tables-for-database
"The active Tables of the Database with `database-id`, in case-insensitive display name order."
[database-id :- ::lib.schema.id/database]
- (t2/select :model/Table :db_id database-id :active true {:order-by [[:%lower.display_name :asc]]}))
+ (t2/select :model/Table :db_id database-id :active true {:from [(warehouse-schema-overlay/table-query)]
+ :order-by [[:%lower.display_name :asc]]}))
(mu/defn active-tables-for-databases
"The active Tables of the Databases with `database-ids`, in database then display name order."
@@ -123,17 +125,19 @@
(t2/select :model/Table
:db_id [:in database-ids]
:active true
- {:order-by [[:db_id :asc] [:%lower.display_name :asc]]}))
+ {:from [(warehouse-schema-overlay/table-query)]
+ :order-by [[:db_id :asc] [:%lower.display_name :asc]]}))
(mu/defn active-table-ids-for-database
"The ids of the active Tables of the Database with `database-id`, or nil."
[database-id :- ::lib.schema.id/database]
- (t2/select-pks-set :model/Table, :db_id database-id, :active true))
+ (t2/select-pks-set :model/Table, :db_id database-id, :active true {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))
(mu/defn pk-fields-for-tables
"The primary-key Fields of the Tables with `table-ids`."
[table-ids :- [:set ::lib.schema.id/table]]
- (t2/select :model/Field, :table_id [:in table-ids], :semantic_type (mdb/isa :type/PK)))
+ (t2/select :model/Field :table_id [:in table-ids] :semantic_type (mdb/isa :type/PK)
+ {:from [(warehouse-schema-overlay/field-query)]}))
(mu/defn databases-for-serdes-reducible
"A reducible of the Databases to export via serdes: routing destinations and the sample database are always
@@ -155,4 +159,4 @@
(mu/defn table-database-id
"The Database id of the Table with `table-id`, or nil."
[table-id :- ::lib.schema.id/table]
- (t2/select-one-fn :db_id :model/Table, :id table-id))
+ (t2/select-one-fn :db_id :model/Table, :id table-id {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))
diff --git a/src/metabase/warehouses_rest/db.clj b/src/metabase/warehouses_rest/db.clj
index 95bdfa42eb7..27aa200e0da 100644
--- a/src/metabase/warehouses_rest/db.clj
+++ b/src/metabase/warehouses_rest/db.clj
@@ -11,6 +11,7 @@
[metabase.util.honey-sql-2 :as h2x]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[metabase.warehouses.schema :as warehouses.schema]
[toucan2.core :as t2]))
@@ -21,14 +22,15 @@
:active true
:db_id [:in database-ids]
:visibility_type nil
- {:order-by [[:%lower.schema :asc]
+ {:from [(warehouse-schema-overlay/table-query)]
+ :order-by [[:%lower.schema :asc]
[:%lower.display_name :asc]]}))
(mu/defn active-visible-schemas-for-databases
"The distinct Database id and schema of the active, visible Tables of the Databases with `database-ids`."
[database-ids :- [:sequential ::lib.schema.id/database]]
(t2/query {:select-distinct [:db_id :schema]
- :from [(t2/table-name :model/Table)]
+ :from [(warehouse-schema-overlay/table-query)]
:where [:and
[:in :db_id database-ids]
[:= :active true]
@@ -124,7 +126,8 @@
like-pattern :- [:or :string vector?]
limit :- ms/PositiveInt]
(t2/select [:model/Table :id :db_id :schema :name]
- {:where [:and [:= :db_id database-id]
+ {:from [(warehouse-schema-overlay/table-query)]
+ :where [:and [:= :db_id database-id]
[:= :active true]
[:like :%lower.name like-pattern]
[:= :visibility_type nil]]
@@ -203,24 +206,26 @@
:%lower.metabase_field/name [:like like-pattern]
:metabase_field.visibility_type [:not-in ["sensitive" "retired"]]
:table.db_id database-id
- {:order-by [[[:lower :metabase_field.name] :asc]
+ {:from [(warehouse-schema-overlay/field-query)]
+ :order-by [[[:lower :metabase_field.name] :asc]
[[:lower :table.name] :asc]]
;; checking for table.active in join makes query faster when there are a lot of inactive tables
- :inner-join [[:metabase_table :table] [:and :table.active
- [:= :table.id :metabase_field.table_id]]]
+ :inner-join [(warehouse-schema-overlay/table-query {:alias :table}) [:and :table.active
+ [:= :table.id :metabase_field.table_id]]]
:limit limit}))
(mu/defn table-ids-for-database
"The ids of the Tables of the Database with `database-id`."
[database-id :- ::lib.schema.id/database]
- (t2/select-fn-set :id :model/Table, :db_id database-id))
+ (t2/select-fn-set :id :model/Table, :db_id database-id {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))
(mu/defn non-sensitive-fields-for-tables
"The id, name, display name, Table id, and types of the non-sensitive Fields of the Tables with `table-ids`."
[table-ids :- [:set ::lib.schema.id/table]]
(t2/select [:model/Field :id :name :display_name :table_id :base_type :semantic_type]
:table_id [:in table-ids]
- :visibility_type [:not-in ["sensitive" "retired"]]))
+ :visibility_type [:not-in ["sensitive" "retired"]]
+ {:from [(warehouse-schema-overlay/field-query)]}))
(mu/defn insert-database!
"Insert the Database `row` and return the inserted instance."
@@ -268,6 +273,48 @@
:right-join [[:metabase_table :t] [:= :f.table_id :t.id]]
:where [:= :t.db_id database-id]}]}))
+(mu/defn active-tables-for-database
+ "The active Tables of the Database with `database-id`."
+ [database-id :- ::lib.schema.id/database]
+ (t2/select :model/Table :db_id database-id :active true {:from [(warehouse-schema-overlay/table-query)]}))
+
+(mu/defn active-table-schemas
+ "The distinct schemas of the active Tables of the Database with `database-id`, in schema order. When
+ `include-hidden?` is false, restricted to Tables with no `visibility_type` (a non-nil value means the Table is
+ hidden -- see [[metabase.warehouse-schema.models.table/visibility-types]])."
+ [database-id :- ::lib.schema.id/database
+ include-hidden? :- :boolean]
+ (let [clauses (cond-> []
+ (not include-hidden?) (conj [:= :visibility_type nil]))]
+ (t2/select-fn-set :schema :model/Table :db_id database-id :active true
+ (merge {:from [(warehouse-schema-overlay/table-query)]
+ :order-by [[:%lower.schema :asc]]}
+ (when clauses
+ {:where (into [:and] clauses)})))))
+
+(mu/defn active-tables-in-schema
+ "The active Tables in `schema` of the Database with `database-id`, in display name order."
+ [database-id :- ::lib.schema.id/database
+ schema :- [:maybe :string]]
+ (t2/select :model/Table
+ :db_id database-id
+ :schema schema
+ :active true
+ {:from [(warehouse-schema-overlay/table-query)]
+ :order-by [[:display_name :asc]]}))
+
+(mu/defn active-visible-tables-in-schema
+ "The active, visible Tables in `schema` of the Database with `database-id`, in display name order."
+ [database-id :- ::lib.schema.id/database
+ schema :- [:maybe :string]]
+ (t2/select :model/Table
+ :db_id database-id
+ :schema schema
+ :active true
+ :visibility_type nil
+ {:from [(warehouse-schema-overlay/table-query)]
+ :order-by [[:display_name :asc]]}))
+
(mu/defn collection-ids-named
"The ids of the Collections named `collection-name`, or nil."
[collection-name :- :string]
diff --git a/src/metabase/xrays/db.clj b/src/metabase/xrays/db.clj
index b1dab7b6918..6a4860459b0 100644
--- a/src/metabase/xrays/db.clj
+++ b/src/metabase/xrays/db.clj
@@ -7,6 +7,7 @@
[metabase.queries.schema :as queries.schema]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
(mu/defn database
@@ -18,19 +19,19 @@
"The Table with `table-id`, or nil. `table-id` may be nil (some callers pass a Card's or Metric's possibly-absent
table id), in which case this returns nil."
[table-id :- [:maybe ::lib.schema.id/table]]
- (t2/select-one :model/Table :id table-id))
+ (t2/select-one :model/Table :id table-id {:from [(warehouse-schema-overlay/table-query)]}))
(mu/defn tables
"The Tables with `table-ids`."
[table-ids :- [:set ::lib.schema.id/table]]
- (t2/select :model/Table :id [:in table-ids]))
+ (t2/select :model/Table :id [:in table-ids] {:from [(warehouse-schema-overlay/table-query)]}))
(mu/defn tables-in-schema
"The Tables in `schema` of the Database with `database-id`. `schema` may be nil (some drivers have no
schema concept)."
[database-id :- ::lib.schema.id/database
schema :- [:maybe :string]]
- (t2/select :model/Table :db_id database-id :schema schema))
+ (t2/select :model/Table :db_id database-id :schema schema {:from [(warehouse-schema-overlay/table-query)]}))
(mu/defn sibling-tables
"The active, visible Tables in `schema` of the Database with `database-id` other than `table-id`. `schema`
@@ -43,7 +44,7 @@
:schema schema
:id [:not= table-id]
:visibility_type nil
- :active true))
+ :active true {:from [(warehouse-schema-overlay/table-query)]}))
(mu/defn candidate-tables-with-field-stats
"The id, schema, name, entity type, Database, field count, and list-likeness of the active, visible Tables of the
@@ -55,7 +56,8 @@
[[:and
[:>= :ts.count 2]
[:= :ts.count_non_pks 1]] :list-like?]]
- {:inner-join [[^:allow-subquery {:select [:f.table_id
+ {:from [(warehouse-schema-overlay/table-query)]
+ :inner-join [[^:allow-subquery {:select [:f.table_id
[:%count.* "count"]
[[:count [:case [:or [:not= :semantic_type "type/PK"]
[:= :f.semantic_type nil]]
@@ -64,7 +66,7 @@
[[:count [:case [:in :f.semantic_type ["type/PK" "type/FK"]]
[:inline 1] :else [:inline nil]]]
:count_pks_and_fks]]
- :from [[:metabase_field :f]]
+ :from [(warehouse-schema-overlay/field-query {:alias :f})]
:where [:= :f.active true]
:group-by [:f.table_id]} :ts]
[:and [:= :ts.table_id :id]
@@ -79,12 +81,12 @@
(mu/defn field
"The Field with `field-id`, or nil."
[field-id :- ::lib.schema.id/field]
- (t2/select-one :model/Field :id field-id))
+ (t2/select-one :model/Field :id field-id {:from [(warehouse-schema-overlay/field-query)]}))
(mu/defn field-name
"The name of the Field with `field-id`, or nil."
[field-id :- ::lib.schema.id/field]
- (t2/select-one-fn :name :model/Field :id field-id))
+ (t2/select-one-fn :name :model/Field :id field-id {:from [(warehouse-schema-overlay/field-query {:user-settings? false})]}))
(mu/defn metadata-column
"The `:metadata/column` with `field-id`, or nil."
@@ -94,17 +96,17 @@
(mu/defn fields-targeting
"The Fields whose FK target is the Field with `field-id`."
[field-id :- ::lib.schema.id/field]
- (t2/select :model/Field :fk_target_field_id field-id))
+ (t2/select :model/Field :fk_target_field_id field-id {:from [(warehouse-schema-overlay/field-query)]}))
(mu/defn fk-fields-for-tables
"The FK Fields of the Tables with `table-ids`."
[table-ids :- [:set ::lib.schema.id/table]]
- (t2/select :model/Field :fk_target_field_id [:not= nil] :table_id [:in table-ids]))
+ (t2/select :model/Field :fk_target_field_id [:not= nil] :table_id [:in table-ids] {:from [(warehouse-schema-overlay/field-query)]}))
(mu/defn active-fk-fields-for-table
"The active FK Fields of the Table with `table-id`."
[table-id :- ::lib.schema.id/table]
- (t2/select :model/Field :table_id table-id :fk_target_field_id [:not= nil] :active true))
+ (t2/select :model/Field :table_id table-id :fk_target_field_id [:not= nil] :active true {:from [(warehouse-schema-overlay/field-query)]}))
(mu/defn fk-target-field-ids-for-table
"The FK target Field ids of the active Fields of the Table with `table-id`."
@@ -112,17 +114,18 @@
(t2/select-fn-set :fk_target_field_id :model/Field
:table_id table-id
:fk_target_field_id [:not= nil]
- :active true))
+ :active true
+ {:from [(warehouse-schema-overlay/field-query)]}))
(mu/defn active-field-ids-for-table
"The ids of the active Fields of the Table with `table-id`."
[table-id :- ::lib.schema.id/table]
- (t2/select-fn-set :id :model/Field :table_id table-id :active true))
+ (t2/select-fn-set :id :model/Field :table_id table-id :active true {:from [(warehouse-schema-overlay/field-query {:user-settings? false})]}))
(mu/defn table-ids-of-fields-targeting
"The Table ids of the active Fields whose FK target is one of `field-ids`."
[field-ids :- [:set ::lib.schema.id/field]]
- (t2/select-fn-set :table_id :model/Field :fk_target_field_id [:in field-ids] :active true))
+ (t2/select-fn-set :table_id :model/Field :fk_target_field_id [:in field-ids] :active true {:from [(warehouse-schema-overlay/field-query)]}))
(mu/defn visible-fields-for-tables
"The active, normally visible, previewable Fields of the Tables with `table-ids`."
@@ -131,7 +134,8 @@
:table_id [:in table-ids]
:visibility_type "normal"
:preview_display true
- :active true))
+ :active true
+ {:from [(warehouse-schema-overlay/field-query)]}))
(mu/defn other-visible-fields-in-table
"The active, normally visible Fields of the Table with `table-id` other than `field-id`."
@@ -141,7 +145,8 @@
:table_id table-id
:id [:not= field-id]
:visibility_type "normal"
- :active true))
+ :active true
+ {:from [(warehouse-schema-overlay/field-query)]}))
(mu/defn card
"The Card with `card-id`, or nil."
diff --git a/test/metabase/app_db/schema_migrations_test.clj b/test/metabase/app_db/schema_migrations_test.clj
index 73ca3cee3fb..061fde19f6b 100644
--- a/test/metabase/app_db/schema_migrations_test.clj
+++ b/test/metabase/app_db/schema_migrations_test.clj
@@ -3362,3 +3362,47 @@
(testing "the mirror column is nullable"
(t2/update! :metabase_field_user_settings :field_id field-id {:data_sensitivity nil})
(is (nil? (t2/select-one-fn :data_sensitivity :metabase_field_user_settings :field_id field-id))))))))
+
+(deftest backfill-field-user-settings-set-flags-test
+ (testing "v64.2026-09-09T00:00:03: description_set, semantic_type_set and fk_target_field_id_set are backfilled
+ from whether the corresponding column is already non-NULL"
+ (impl/test-migrations ["v64.2026-09-09T00:00:00" "v64.2026-09-09T00:00:03"] [migrate!]
+ (let [db-id (t2/insert-returning-pk! :metabase_database {:name "FUS Flags Test DB"
+ :engine "h2"
+ :created_at :%now
+ :updated_at :%now
+ :details "{}"})
+ table-id (t2/insert-returning-pk! :metabase_table {:active true
+ :db_id db-id
+ :name "a table"
+ :created_at :%now
+ :updated_at :%now})
+ insert-field! (fn [name]
+ (t2/insert-returning-pk! :metabase_field {:table_id table-id
+ :name name
+ :active true
+ :base_type "type/Text"
+ :database_type "TEXT"
+ :created_at :%now
+ :updated_at :%now}))
+ target-id (insert-field! "target")
+ all-set-id (insert-field! "all_set")
+ none-set-id (insert-field! "none_set")
+ mixed-id (insert-field! "mixed")]
+ (t2/insert! :metabase_field_user_settings {:field_id all-set-id
+ :description "a description"
+ :semantic_type "type/Category"
+ :fk_target_field_id target-id})
+ (t2/insert! :metabase_field_user_settings {:field_id none-set-id})
+ (t2/insert! :metabase_field_user_settings {:field_id mixed-id
+ :description "only description is set"})
+ (migrate!)
+ (testing "every column set is flagged true"
+ (is (=? {:description_set true, :semantic_type_set true, :fk_target_field_id_set true}
+ (t2/select-one :metabase_field_user_settings :field_id all-set-id))))
+ (testing "every column NULL is flagged false"
+ (is (=? {:description_set false, :semantic_type_set false, :fk_target_field_id_set false}
+ (t2/select-one :metabase_field_user_settings :field_id none-set-id))))
+ (testing "only the columns that are non-NULL are flagged true"
+ (is (=? {:description_set true, :semantic_type_set false, :fk_target_field_id_set false}
+ (t2/select-one :metabase_field_user_settings :field_id mixed-id))))))))
diff --git a/test/metabase/core/modules_consistency_test.clj b/test/metabase/core/modules_consistency_test.clj
index 77fe45ede6e..06870ed649d 100644
--- a/test/metabase/core/modules_consistency_test.clj
+++ b/test/metabase/core/modules_consistency_test.clj
@@ -24,6 +24,12 @@
nil)))
["./bin/bb" "bb"])))
+(def ^:private bb-env
+ "The environment Babashka runs under: a writable classpath cache, wherever the runner's home directory is."
+ (delay (assoc (into {} (System/getenv))
+ "CLJ_CACHE" (str (java.nio.file.Files/createTempDirectory
+ "bb-cpcache" (into-array java.nio.file.attribute.FileAttribute []))))))
+
(defn- bb-eval
"Evaluate `form` with `mage.modules` loaded in Babashka."
[form]
@@ -32,7 +38,7 @@
(throw (ex-info "No working babashka at ./bin/bb or on PATH. Run any ./bin/mage task once to install it."
{})))
expr (pr-str `(do (require 'mage.modules) (~'prn ~form)))
- {:keys [exit out err]} (shell/sh bb "-e" expr)]
+ {:keys [exit out err]} (shell/sh bb "-e" expr :env @bb-env)]
(when-not (zero? exit)
(throw (ex-info "babashka evaluation failed" {:exit exit, :stderr err, :expr expr})))
(edn/read-string out)))
diff --git a/test/metabase/data_studio/api/table_test.clj b/test/metabase/data_studio/api/table_test.clj
index 3a21620e855..d5863fda8cc 100644
--- a/test/metabase/data_studio/api/table_test.clj
+++ b/test/metabase/data_studio/api/table_test.clj
@@ -9,6 +9,7 @@
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
[metabase.util.quick-task :as quick-task]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2])
(:import (java.util.concurrent CountDownLatch Executors TimeUnit)))
@@ -16,6 +17,12 @@
(use-fixtures :once (fixtures/initialize :db))
+(defn- user-table-fn
+ "The value of `column` on the Table with `table-id` as users see it. A bare `t2/select :model/Table` shows sync's."
+ [column table-id]
+ (t2/select-one-fn column :model/Table :id table-id
+ {:from [(warehouse-schema-overlay/table-query)]}))
+
(deftest bulk-edit-visibility-sync-test
(testing "POST /api/data-studio/table/edit visibility field synchronization"
(mt/with-temp [:model/Database {db-id :id} {}
@@ -26,25 +33,25 @@
(mt/user-http-request :crowberto :post 200 "data-studio/table/edit"
{:table_ids [table-1-id table-2-id]
:data_layer "internal"})
- (is (= :internal (t2/select-one-fn :data_layer :model/Table :id table-1-id)))
- (is (= nil (t2/select-one-fn :visibility_type :model/Table :id table-1-id)))
- (is (= :internal (t2/select-one-fn :data_layer :model/Table :id table-2-id)))
- (is (= nil (t2/select-one-fn :visibility_type :model/Table :id table-2-id))))
+ (is (= :internal (user-table-fn :data_layer table-1-id)))
+ (is (= nil (user-table-fn :visibility_type table-1-id)))
+ (is (= :internal (user-table-fn :data_layer table-2-id)))
+ (is (= nil (user-table-fn :visibility_type table-2-id))))
(testing "updating data_layer to hidden syncs to hidden visibility_type"
;; Update one table back to hidden, which should sync to :hidden
(mt/user-http-request :crowberto :post 200 "data-studio/table/edit"
{:table_ids [table-1-id]
:data_layer "hidden"})
- (is (= :hidden (t2/select-one-fn :data_layer :model/Table :id table-1-id)))
- (is (= :hidden (t2/select-one-fn :visibility_type :model/Table :id table-1-id))))
+ (is (= :hidden (user-table-fn :data_layer table-1-id)))
+ (is (= :hidden (user-table-fn :visibility_type table-1-id))))
(testing "visibility_type is not part of this endpoint, so it is dropped and data_layer alone applies"
(mt/user-http-request :crowberto :post 200 "data-studio/table/edit"
{:table_ids [table-1-id]
:visibility_type "hidden"
:data_layer "final"})
;; had visibility_type been honoured, the model would have refused to update both at once
- (is (= :final (t2/select-one-fn :data_layer :model/Table :id table-1-id)))
- (is (= nil (t2/select-one-fn :visibility_type :model/Table :id table-1-id)))))))
+ (is (= :final (user-table-fn :data_layer table-1-id)))
+ (is (= nil (user-table-fn :visibility_type table-1-id)))))))
(deftest bulk-edit-does-not-allow-changing-data-source-away-from-transform-test
(testing "POST /api/data-studio/table/edit cannot change a transform-created table's data_source"
@@ -53,7 +60,7 @@
(mt/user-http-request :crowberto :post 400 "data-studio/table/edit"
{:table_ids [table-id]
:data_source "ingested"})
- (is (= :metabase-transform (t2/select-one-fn :data_source :model/Table :id table-id))))))
+ (is (= :metabase-transform (user-table-fn :data_source table-id))))))
(deftest data-analyst-can-access-endpoints-test
(testing "Data analysts (members of Data Analysts group) can access data studio endpoints"
@@ -304,9 +311,9 @@
:data_layer "hidden"
:data_authority "authoritative"
:data_source "ingested"})
- (is (= #{:hidden} (t2/select-fn-set :data_layer :model/Table :db_id [:in [clojure jvm]])))
- (is (= #{:authoritative} (t2/select-fn-set :data_authority :model/Table :db_id [:in [clojure jvm]])))
- (is (= #{:ingested} (t2/select-fn-set :data_source :model/Table :db_id [:in [clojure jvm]]))))
+ (is (= #{:hidden} (t2/select-fn-set :data_layer :model/Table :db_id [:in [clojure jvm]] {:from [(warehouse-schema-overlay/table-query)]})))
+ (is (= #{:authoritative} (t2/select-fn-set :data_authority :model/Table :db_id [:in [clojure jvm]] {:from [(warehouse-schema-overlay/table-query)]})))
+ (is (= #{:ingested} (t2/select-fn-set :data_source :model/Table :db_id [:in [clojure jvm]] {:from [(warehouse-schema-overlay/table-query)]}))))
(testing "updating with all selectors"
(mt/user-http-request :crowberto :post 200 "data-studio/table/edit"
{:database_ids [clojure]
@@ -319,9 +326,9 @@
classes :internal
gc :internal
jit :internal}
- (t2/select-pk->fn :data_layer :model/Table :db_id [:in [clojure jvm]]))))
+ (t2/select-pk->fn :data_layer :model/Table :db_id [:in [clojure jvm]] {:from [(warehouse-schema-overlay/table-query)]}))))
(testing "can update owner_email"
- (is (= #{nil} (t2/select-fn-set :owner_email :model/Table :db_id [:in [clojure jvm]])))
+ (is (= #{nil} (t2/select-fn-set :owner_email :model/Table :db_id [:in [clojure jvm]] {:from [(warehouse-schema-overlay/table-query)]})))
(mt/user-http-request :crowberto :post 200 "data-studio/table/edit"
{:database_ids [clojure]
:owner_email "[email protected]"})
@@ -331,9 +338,9 @@
classes nil
gc nil
jit nil}
- (t2/select-pk->fn :owner_email :model/Table :db_id [:in [clojure jvm]]))))
+ (t2/select-pk->fn :owner_email :model/Table :db_id [:in [clojure jvm]] {:from [(warehouse-schema-overlay/table-query)]}))))
(testing "can update owner_user_id"
- (is (= #{nil} (t2/select-fn-set :owner_user_id :model/Table :db_id [:in [clojure jvm]])))
+ (is (= #{nil} (t2/select-fn-set :owner_user_id :model/Table :db_id [:in [clojure jvm]] {:from [(warehouse-schema-overlay/table-query)]})))
(mt/user-http-request :crowberto :post 200 "data-studio/table/edit"
{:table_ids [beans classes]
:owner_user_id (mt/user->id :rasta)})
@@ -343,7 +350,7 @@
classes (mt/user->id :rasta)
gc nil
jit nil}
- (t2/select-pk->fn :owner_user_id :model/Table :db_id [:in [clojure jvm]])))))))
+ (t2/select-pk->fn :owner_user_id :model/Table :db_id [:in [clojure jvm]] {:from [(warehouse-schema-overlay/table-query)]})))))))
;;; ------------------------------------------------- Selection Tests -------------------------------------------------
diff --git a/test/metabase/driver/db_test.clj b/test/metabase/driver/db_test.clj
new file mode 100644
index 00000000000..86cbc66a885
--- /dev/null
+++ b/test/metabase/driver/db_test.clj
@@ -0,0 +1,15 @@
+(ns metabase.driver.db-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.driver.db :as driver.db]
+ [metabase.test :as mt]))
+
+(deftest json-field-names-with-unfolding-disabled-honors-user-setting-test
+ (testing "json-field-names-with-unfolding-disabled honors a user-set json_unfolding that differs from the sync value"
+ (mt/with-temp [:model/Table {table-id :id} {}
+ :model/Field {field-id :id} {:table_id table-id, :name "PAYLOAD"
+ :base_type :type/JSON, :json_unfolding true}]
+ (testing "sanity check: unfolding is enabled by sync"
+ (is (empty? (driver.db/json-field-names-with-unfolding-disabled table-id))))
+ (mt/with-temp [:model/FieldUserSettings _ {:field_id field-id, :json_unfolding false}]
+ (is (= #{"PAYLOAD"} (driver.db/json-field-names-with-unfolding-disabled table-id)))))))
diff --git a/test/metabase/parameters/chain_filter_test.clj b/test/metabase/parameters/chain_filter_test.clj
index 9586013f2f9..25bd223cebb 100644
--- a/test/metabase/parameters/chain_filter_test.clj
+++ b/test/metabase/parameters/chain_filter_test.clj
@@ -502,6 +502,19 @@
(is (= (mt/id :people :name)
(#'chain-filter/remapped-field-id (mt/id :orders :user_id)))))
+(deftest fk-field-to-pk-field-to-name-field-remapped-field-id-user-override-test
+ (testing "Implicit FK->PK->Name remapping honors a user-set FK target that differs from the sync value"
+ (mt/with-temp-vals-in-db :model/Field (mt/id :orders :user_id) {:semantic_type nil, :fk_target_field_id nil}
+ (testing "sanity check: with the sync FK cleared, there's no remapping"
+ (is (nil? (#'chain-filter/remapped-field-id (mt/id :orders :user_id)))))
+ (mt/with-temp [:model/FieldUserSettings _ {:field_id (mt/id :orders :user_id)
+ :semantic_type :type/FK
+ :semantic_type_set true
+ :fk_target_field_id (mt/id :people :id)
+ :fk_target_field_id_set true}]
+ (is (= (mt/id :people :name)
+ (#'chain-filter/remapped-field-id (mt/id :orders :user_id))))))))
+
(deftest ^:parallel field-to-field-remapped-chain-filter-test
(testing "Field-to-field remapping: venues.category_id -> categories.name\n"
(testing "Show me venue IDs (names)"
diff --git a/test/metabase/permissions/db_test.clj b/test/metabase/permissions/db_test.clj
new file mode 100644
index 00000000000..0a31d0ceeed
--- /dev/null
+++ b/test/metabase/permissions/db_test.clj
@@ -0,0 +1,16 @@
+(ns metabase.permissions.db-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.permissions.db :as permissions.db]
+ [metabase.test :as mt]))
+
+(deftest field-visibility-info-honors-user-set-visibility-type-test
+ (testing "field-visibility-info honors a user-set visibility_type (e.g. sensitive) over the sync value"
+ (mt/with-temp [:model/Table {table-id :id} {}
+ :model/Field {field-id :id} {:table_id table-id, :visibility_type "normal"}]
+ (testing "sanity check: the sync value shows by default"
+ (is (=? {:id field-id, :visibility_type :normal, :table_id table-id}
+ (first (permissions.db/field-visibility-info #{field-id})))))
+ (mt/with-temp [:model/FieldUserSettings _ {:field_id field-id, :visibility_type :sensitive}]
+ (is (=? {:id field-id, :visibility_type :sensitive, :table_id table-id}
+ (first (permissions.db/field-visibility-info #{field-id}))))))))
diff --git a/test/metabase/search/spec_test.clj b/test/metabase/search/spec_test.clj
index cd58cd3d9ff..2205e95924c 100644
--- a/test/metabase/search/spec_test.clj
+++ b/test/metabase/search/spec_test.clj
@@ -118,7 +118,10 @@
:where [:= :updated.id :this.id]}
{:search-model "table"
:fields #{:authority_level :name :type :location}
- :where [:and [:= :this.is_published true] [:= :updated.id :this.collection_id]]}}}
+ :where [:and [:= :this.is_published true] [:= :updated.id :this.collection_id]]}}
+ :TableUserSettings #{{:search-model "table"
+ :fields #{:updated_at}
+ :where [:= :updated.table_id :this.id]}}}
(#'search.spec/merge-hooks
[(#'search.spec/search-model-hooks (search.spec/spec "table"))
(#'search.spec/search-model-hooks (search.spec/spec "segment"))
diff --git a/test/metabase/sync/analyze/data_sensitivity_test.clj b/test/metabase/sync/analyze/data_sensitivity_test.clj
index 2337cd4acf5..45536596436 100644
--- a/test/metabase/sync/analyze/data_sensitivity_test.clj
+++ b/test/metabase/sync/analyze/data_sensitivity_test.clj
@@ -6,10 +6,19 @@
[metabase.sync.core :as sync]
[metabase.test :as mt]
[metabase.util :as u]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[metabase.warehouse-schema.models.field-user-settings :as field-user-settings]
[toucan2.core :as t2]))
-(defn- label [field-or-id]
+(defn- label
+ "The `data_sensitivity` a user sees: their own label when they set one, else the classifier's."
+ [field-or-id]
+ (t2/select-one-fn :data_sensitivity :model/Field :id (u/the-id field-or-id)
+ {:from [(warehouse-schema-overlay/field-query {:alias :f})]}))
+
+(defn- sync-label
+ "The `data_sensitivity` the classifier itself wrote to `metabase_field`."
+ [field-or-id]
(t2/select-one-fn :data_sensitivity :model/Field :id (u/the-id field-or-id)))
(defn- mirror-label [field-or-id]
@@ -110,24 +119,32 @@
(is (= :PII (label ssn))))))))
(deftest user-label-in-mirror-wins-test
- (testing "a mirror-only user label is what the field reads after the classifier writes"
+ (testing "a mirror-only user label survives the classifier writing its own inference to the raw Field"
(mt/with-temp [:model/Database db {}
:model/Table table {:db_id (:id db) :name "app_users"}
:model/Field ssn {:table_id (:id table) :name "ssn" :base_type :type/Text}
:model/Field notes {:table_id (:id table) :name "notes" :base_type :type/Text}]
(field-user-settings/upsert-user-settings ssn {:data_sensitivity :PUBLIC})
(field-user-settings/upsert-user-settings notes {:data_sensitivity :PHI})
- (is (nil? (label ssn)))
- (is (nil? (label notes)))
- (testing "stats count the rule result, the overlay decides what is stored"
+ (testing "before the scan only the user's label exists"
+ (is (nil? (sync-label ssn)))
+ (is (nil? (sync-label notes)))
+ (is (= :PUBLIC (label ssn)))
+ (is (= :PHI (label notes))))
+ (testing "the scan still classifies both fields on the raw Field"
(is (= {:fields-scanned 2 :fields-labeled 1 :fields-failed 0}
(sync.data-sensitivity/scan-data-sensitivity! db))))
- (is (= :PUBLIC (label ssn)))
- (is (= :PUBLIC (mirror-label ssn)))
- (is (= :PHI (label notes)))
- (is (= :PHI (mirror-label notes)))
- (testing "a forced rescan still cannot override the user's PUBLIC"
+ (testing "the classifier wrote its own label, and the user's still wins on read"
+ (is (= :PII (sync-label ssn)))
+ (is (= :PUBLIC (mirror-label ssn)))
+ (is (= :PUBLIC (label ssn)))
+ (is (= :PUBLIC (sync-label notes)))
+ (is (= :PHI (mirror-label notes)))
+ (is (= :PHI (label notes))))
+ (testing "a forced rescan does not revisit a categorized Field, mirror or not"
(sync.data-sensitivity/scan-data-sensitivity! db :force? true)
+ (is (= :PII (sync-label ssn)))
+ (is (= :PUBLIC (mirror-label ssn)))
(is (= :PUBLIC (label ssn)))))))
(deftest reset-data-sensitivity-test
@@ -146,20 +163,21 @@
(t2/insert! :model/FieldUserSettings {:field_id (:id email)})
(sync.data-sensitivity/scan-data-sensitivity! db)
(sync.data-sensitivity/scan-data-sensitivity! other-db)
- (is (= [:PII :PUBLIC :PHI :PII :PUBLIC :PII]
- (map label [ssn foo notes email total far])))
+ (is (= [:PII :PUBLIC :PUBLIC :PII :PUBLIC :PII]
+ (map sync-label [ssn foo notes email total far])))
(testing "a table scope clears only that table's classifier labels"
(is (= 1 (sync.data-sensitivity/reset-data-sensitivity! orders)))
- (is (nil? (label total)))
- (is (= :PII (label ssn))))
+ (is (nil? (sync-label total)))
+ (is (= :PII (sync-label ssn))))
(testing "a database scope clears categories and PUBLIC alike, including fields with a label-less mirror row"
(is (= 3 (sync.data-sensitivity/reset-data-sensitivity! db)))
- (is (= [nil nil nil] (map label [ssn foo email]))))
- (testing "a label backed by the mirror is human-set and survives"
- (is (= :PHI (label notes)))
- (is (= :PHI (mirror-label notes))))
+ (is (= [nil nil nil] (map sync-label [ssn foo email]))))
+ (testing "a label backed by the mirror is human-set; the reset clears only the classifier's own label"
+ (is (= :PUBLIC (sync-label notes)))
+ (is (= :PHI (mirror-label notes)))
+ (is (= :PHI (label notes))))
(testing "other databases are untouched"
- (is (= :PII (label far))))
+ (is (= :PII (sync-label far))))
(testing "a second reset finds nothing"
(is (zero? (sync.data-sensitivity/reset-data-sensitivity! db))))))
@@ -170,18 +188,19 @@
:model/Field notes {:table_id (:id table) :name "notes" :base_type :type/Text}]
(field-user-settings/upsert-user-settings notes {:data_sensitivity :PHI})
(sync.data-sensitivity/scan-data-sensitivity! db)
- (is (= :PII (label ssn)))
+ (is (= :PII (sync-label ssn)))
(with-redefs [analyze/infer-data-sensitivity (constantly :SEC_KEY)]
(testing "a forced scan leaves a categorized field on its old label"
(sync.data-sensitivity/scan-data-sensitivity! db :force? true)
- (is (= :PII (label ssn))))
+ (is (= :PII (sync-label ssn))))
(testing "a reset scan recomputes it under the current rules and reports the reset count"
(is (= {:fields-scanned 1 :fields-labeled 1 :fields-failed 0 :fields-reset 1}
(sync.data-sensitivity/scan-data-sensitivity! db :reset? true)))
- (is (= :SEC_KEY (label ssn))))
- (testing "the user's label is neither reset nor rescanned"
- (is (= :PHI (label notes)))
- (is (= :PHI (mirror-label notes)))))))
+ (is (= :SEC_KEY (sync-label ssn))))
+ (testing "the user's label in the mirror is untouched, and is still what readers see"
+ (is (= :SEC_KEY (sync-label notes)) "reclassified by the earlier forced scan, which also revisits PUBLIC fields")
+ (is (= :PHI (mirror-label notes)))
+ (is (= :PHI (label notes)))))))
(deftest classification-failure-is-counted-and-retried-test
(mt/with-temp [:model/Database db {}
diff --git a/test/metabase/sync/field_values_test.clj b/test/metabase/sync/field_values_test.clj
index d37550b7e64..89319361df2 100644
--- a/test/metabase/sync/field_values_test.clj
+++ b/test/metabase/sync/field_values_test.clj
@@ -6,6 +6,7 @@
[java-time.api :as t]
[metabase.analyze.core :as analyze]
[metabase.sync.core :as sync]
+ [metabase.sync.db :as sync.db]
[metabase.sync.field-values :as sync.field-values]
[metabase.sync.util-test :as sync.util-test]
[metabase.test :as mt]
@@ -321,6 +322,17 @@
;;; ---------------------------------- can-batch-distinct? ----------------------------------
+(deftest fields-for-field-values-honors-user-settings-test
+ (testing "fields-for-field-values reflects the user's has_field_values/visibility_type overrides"
+ (mt/with-temp [:model/Table {table-id :id} {}
+ :model/Field {field-id :id} {:table_id table-id, :has_field_values nil, :visibility_type "normal"}]
+ (testing "sanity check: the sync values show by default"
+ (is (=? {:has_field_values nil, :visibility_type :normal}
+ (first (sync.db/fields-for-field-values [field-id])))))
+ (mt/with-temp [:model/FieldUserSettings _ {:field_id field-id, :has_field_values :none, :visibility_type :sensitive}]
+ (is (=? {:has_field_values :none, :visibility_type :sensitive}
+ (first (sync.db/fields-for-field-values [field-id]))))))))
+
(deftest can-batch-distinct?-test
(testing "SQL driver with :nested-queries and no required filter → batch path"
(mt/with-temp [:model/Database {db-id :id} {:engine :h2}
diff --git a/test/metabase/sync/sync_metadata/comments_test.clj b/test/metabase/sync/sync_metadata/comments_test.clj
index bd3f131a771..787ac2575dd 100644
--- a/test/metabase/sync/sync_metadata/comments_test.clj
+++ b/test/metabase/sync/sync_metadata/comments_test.clj
@@ -11,9 +11,13 @@
[metabase.test.data.interface :as tx]
[metabase.test.data.sql :as sql.tx]
[metabase.util :as u]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
+ [metabase.warehouse-schema.models.field-user-settings :as field-user-settings]
[toucan2.core :as t2]))
-(defn- db->fields [db]
+(defn- db->fields
+ "The name and description sync itself recorded for each Field of `db`, ignoring any description a user set."
+ [db]
(let [table-ids (t2/select-pks-set :model/Table :db_id (u/the-id db))]
(set (map (partial into {}) (t2/select ['Field :name :description] :table_id [:in table-ids])))))
@@ -47,18 +51,20 @@
[["foo"]]]])
(deftest comment-should-not-overwrite-custom-description-test
- (testing (str "test changing the description in metabase db so we can check it is not overwritten by comment in "
- "source db when resyncing")
+ (testing "a user-set description lives in FieldUserSettings and is never overwritten by resyncing, while the raw
+ Field still picks up the comment from the source db"
(mt/test-drivers (mt/normal-driver-select {:+features [::field-comments-sync]})
(mt/dataset update-desc
(mt/with-temp-copy-of-db
- ;; change the description in metabase while the source table comment remains the same
- (t2/update! :model/Field {:id (mt/id "update_desc" "updated_desc")}, {:description "updated description"})
- ;; now sync the DB again, this should NOT overwrite the manually updated description
- (sync/sync-table! (t2/select-one :model/Table :id (mt/id "update_desc")))
- (is (= #{{:name (mt/format-name "id"), :description nil}
- {:name (mt/format-name "updated_desc"), :description "updated description"}}
- (db->fields (mt/db)))))))))
+ (let [field-id (mt/id "update_desc" "updated_desc")]
+ (t2/update! :model/Field field-id {:description nil})
+ (field-user-settings/upsert-user-settings {:id field-id} {:description "updated description"})
+ (sync/sync-table! (t2/select-one :model/Table :id (mt/id "update_desc")))
+ (is (= #{{:name (mt/format-name "id"), :description nil}
+ {:name (mt/format-name "updated_desc"), :description "original comment"}}
+ (db->fields (mt/db))))
+ (is (= "updated description"
+ (:description (t2/select-one :model/Field :id field-id {:from [(warehouse-schema-overlay/field-query)]}))))))))))
(tx/defdataset ^:private comment-after-sync
[["comment_after_sync"
diff --git a/test/metabase/sync/sync_metadata/fields/sync_metadata_test.clj b/test/metabase/sync/sync_metadata/fields/sync_metadata_test.clj
index f552780045c..b96672a49c5 100644
--- a/test/metabase/sync/sync_metadata/fields/sync_metadata_test.clj
+++ b/test/metabase/sync/sync_metadata/fields/sync_metadata_test.clj
@@ -6,6 +6,7 @@
[metabase.sync.sync-metadata.fields.sync-metadata :as sync-metadata]
[metabase.test :as mt]
[metabase.util :as u]
+ [metabase.warehouse-schema.models.field-user-settings :as field-user-settings]
[next.jdbc :as next.jdbc]
[toucan2.core :as t2]))
@@ -300,28 +301,31 @@
(deftest base-type-change-will-trigger-fingerprint-and-analyze-test
(testing "A base type of a field changes only when the field is dropped then a new field with the name is created (#37047).
In this case, we should make sure effective type is set to base type"
- (is (= [["FieldUserSettings"
- 1
- {:effective_type :type/Text
- :coercion_strategy nil
- :semantic_type nil}]
- ["Field"
- 1
- {:base_type :type/Text
- :effective_type :type/Text
- :coercion_strategy nil
- :fingerprint_version 0
- :fingerprint nil
- :semantic_type nil}]]
- (updates-that-will-be-performed!
- (merge default-metadata
- {:id 1
- :base-type :type/Text
- :effective-type :type/Text})
- (merge default-metadata
- {:id 1
- :base-type :type/Integer
- :effective-type :type/Integer}))))
+ (mt/with-temp [:model/Field {field-id :id} {}]
+ (field-user-settings/upsert-user-settings {:id field-id} {:semantic_type :type/Category})
+ (is (= [["FieldUserSettings"
+ field-id
+ {:effective_type nil
+ :coercion_strategy nil
+ :semantic_type nil
+ :semantic_type_set false}]
+ ["Field"
+ field-id
+ {:base_type :type/Text
+ :effective_type :type/Text
+ :coercion_strategy nil
+ :fingerprint_version 0
+ :fingerprint nil
+ :semantic_type nil}]]
+ (updates-that-will-be-performed!
+ (merge default-metadata
+ {:id field-id
+ :base-type :type/Text
+ :effective-type :type/Text})
+ (merge default-metadata
+ {:id field-id
+ :base-type :type/Integer
+ :effective-type :type/Integer})))))
(testing "and sync will re-fingerprint and analyze this field"
(mt/with-temp-test-data [["table"
[{:field-name "field"
@@ -443,16 +447,18 @@
heal effective_type to match base_type. This catches drifted state from older
Metabase versions that the customer's instance carries forward."
(testing "broken row gets healed even when base_type does not change"
- (is (= [["FieldUserSettings" 1 {:effective_type :type/Number}]
- ["Field" 1 {:effective_type :type/Number}]]
- (updates-that-will-be-performed!
- (merge default-metadata
- {:base-type :type/Number})
- (merge default-metadata
- {:id 1
- :base-type :type/Number
- :effective-type :type/Text
- :coercion-strategy nil})))))
+ (mt/with-temp [:model/Field {field-id :id} {}]
+ (field-user-settings/upsert-user-settings {:id field-id} {:effective_type :type/Text})
+ (is (= [["FieldUserSettings" field-id {:effective_type nil :coercion_strategy nil}]
+ ["Field" field-id {:effective_type :type/Number}]]
+ (updates-that-will-be-performed!
+ (merge default-metadata
+ {:base-type :type/Number})
+ (merge default-metadata
+ {:id field-id
+ :base-type :type/Number
+ :effective-type :type/Text
+ :coercion-strategy nil}))))))
(testing "row with a real coercion_strategy is left alone — effective_type ≠ base_type is legitimate"
(is (= []
(updates-that-will-be-performed!
@@ -490,6 +496,7 @@
(deftest base-type-change-leaves-data-sensitivity-test
(testing "the override write on a base type change does not include data_sensitivity"
(mt/with-temp [:model/Field {field-id :id} {:data_sensitivity :PII}]
+ (field-user-settings/upsert-user-settings {:id field-id} {:semantic_type :type/Category})
(let [updates (updates-that-will-be-performed!
(merge default-metadata
{:id field-id
@@ -523,9 +530,9 @@
(sync/sync-table! (t2/select-one :model/Table (mt/id :table)))
(is (=? {:base_type :type/Integer
:effective_type :type/Integer
- :data_sensitivity :PII}
+ :data_sensitivity nil}
(t2/select-one :model/Field :id field-id)))
- (is (=? {:effective_type :type/Integer
+ (is (=? {:effective_type nil
:data_sensitivity :PII}
(t2/select-one :model/FieldUserSettings :field_id field-id))))
(finally
diff --git a/test/metabase/sync/sync_metadata/fields_test.clj b/test/metabase/sync/sync_metadata/fields_test.clj
index ea9e5e402ae..46f08c7f569 100644
--- a/test/metabase/sync/sync_metadata/fields_test.clj
+++ b/test/metabase/sync/sync_metadata/fields_test.clj
@@ -19,6 +19,8 @@
[metabase.test.data.one-off-dbs :as one-off-dbs]
[metabase.test.mock.toucanery :as toucanery]
[metabase.util :as u]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
+ [metabase.warehouse-schema.models.field-user-settings :as field-user-settings]
[toucan2.connection :as t2.connection]
[toucan2.core :as t2]))
@@ -167,11 +169,14 @@
(mt/user-http-request :crowberto :put 200 (format "field/%d" (:id field)) {:coercion_strategy :Coercion/String->Integer})
(sync/sync-database! db)
(is (=? {:effective_type :type/Integer :coercion_strategy :Coercion/String->Integer}
- (t2/select-one :model/Field :name "string_tbc_int_col")))
+ (t2/select-one :model/Field :id (:id field) {:from [(warehouse-schema-overlay/field-query)]})))
(jdbc/execute! db-spec ["ALTER TABLE \"base_type_change_test\" ALTER COLUMN \"string_tbc_int_col\" TYPE int USING \"string_tbc_int_col\"::integer;"])
(sync/sync-database! db)
- (is (=? {:coercion_strategy nil}
- (t2/select-one :model/Field :name "string_tbc_int_col"))))))))
+ (testing "the base type change unsets the user's coercion, on the Field and for the user"
+ (is (=? {:coercion_strategy nil}
+ (t2/select-one :model/Field :id (:id field))))
+ (is (=? {:coercion_strategy nil :effective_type :type/Integer}
+ (t2/select-one :model/Field :id (:id field))))))))))
(deftest dont-show-deleted-fields-test
(testing "make sure deleted fields doesn't show up in `:fields` of a table"
@@ -305,7 +310,7 @@
{:step-info (sync.util-test/only-step-keys step-info)
:task-details task_details
:semantic-type semantic_type
- :fk-target-exists? (t2/exists? :model/Field :id fk_target_field_id)}))]
+ :fk-target-exists? (t2/exists? :model/Field :id fk_target_field_id {:from [(warehouse-schema-overlay/field-query)]})}))]
(testing "before"
(is (= {:step-info {:total-fks 6, :updated-fks 0, :total-failed 0}
:task-details {:total-fks 6, :updated-fks 0, :total-failed 0}
@@ -321,13 +326,13 @@
(state))))))))
(deftest sync-table-fks-test2
- (testing "Check that sync-table! causes FKs to be left alone if they'd override user-set values"
+ (testing "Check that sync-table! keeps writing FKs to the Field while the user keeps seeing their own values"
(mt/with-temp-copy-of-db
(letfn [(state []
(let [{:keys [step-info]
{:keys [task_details]} :task-history} (sync.util-test/sync-database! "sync-fks" (mt/db))
- {:keys [semantic_type fk_target_field_id]} (t2/select-one [:model/Field :semantic_type :fk_target_field_id]
- :id (mt/id :checkins :user_id))]
+ {:keys [semantic_type fk_target_field_id]} (t2/select-one :model/Field :id (mt/id :checkins :user_id)
+ {:from [(warehouse-schema-overlay/field-query)]})]
{:step-info (sync.util-test/only-step-keys step-info)
:task-details task_details
:semantic-type semantic_type
@@ -344,7 +349,10 @@
:task-details {:total-fks 6, :updated-fks 0, :total-failed 0}
:semantic-type :type/Name
:fk-target-exists? false}
- (state))))))))
+ (state)))
+ (testing "sync's own row still carries the FK it detected, under the user's :type/Name"
+ (is (=? {:semantic_type :type/FK :fk_target_field_id int?}
+ (t2/select-one :model/Field :id (mt/id :checkins :user_id))))))))))
(deftest case-sensitive-conflict-test
(testing "Two columns with same lower-case name can be synced (#17387)"
@@ -528,13 +536,14 @@
(let [table-id (t2/select-one-pk :model/Table :db_id (u/the-id database) :name "test_table")
field-id (t2/select-one-pk :model/Field :table_id table-id :name "something")]
(mt/user-http-request :crowberto :put 200 (format "field/%d" field-id) {:visibility_type :normal})
- (let [field-after-manual-change (t2/select-one :model/Field :id field-id)]
- (is (= :normal (:visibility_type field-after-manual-change))
- "Manual change should set visibility_type to :normal")))
+ (is (= :normal (:visibility_type (t2/select-one :model/Field :id field-id
+ {:from [(warehouse-schema-overlay/field-query)]})))
+ "Manual change should set visibility_type to :normal"))
(sync/sync-database! database)
(let [table-id (t2/select-one-pk :model/Table :db_id (u/the-id database) :name "test_table")
- field-after-second-sync (t2/select-one :model/Field :table_id table-id :name "something")]
- (is (= :normal (:visibility_type field-after-second-sync))
+ field-id (t2/select-one-pk :model/Field :table_id table-id :name "something")]
+ (is (= :normal (:visibility_type (t2/select-one :model/Field :id field-id
+ {:from [(warehouse-schema-overlay/field-query)]})))
"Second sync should preserve manually set :normal visibility_type"))))))))
(deftest user-set-fks-are-preserved-by-sync-test
@@ -553,12 +562,13 @@
(testing "should not have FK relationship"
(is (nil? (:fk_target_field_id flocks-example-bird-name-field)))
(is (not= :type/FK (:semantic_type flocks-example-bird-name-field))))
- (t2/update! :model/Field (u/the-id flocks-example-bird-name-field)
- {:semantic_type :type/FK
- :fk_target_field_id (u/the-id birds-example-name-field)})
+ (field-user-settings/upsert-user-settings
+ {:id (u/the-id flocks-example-bird-name-field)}
+ {:semantic_type :type/FK
+ :fk_target_field_id (u/the-id birds-example-name-field)})
(testing "after sync, user-set FK is preserved"
(sync/sync-database! (mt/db))
- (let [field-after-sync (t2/select-one :model/Field :id (u/the-id flocks-example-bird-name-field))]
+ (let [field-after-sync (t2/select-one :model/Field :id (u/the-id flocks-example-bird-name-field) {:from [(warehouse-schema-overlay/field-query)]})]
(is (= :type/FK (:semantic_type field-after-sync)))
(is (= (u/the-id birds-example-name-field) (:fk_target_field_id field-after-sync)))))))))
@@ -593,7 +603,7 @@
(is (false? (:database_is_nullable b)))))))
(deftest data-sensitivity-survives-sync-test
- (testing "a user-set data_sensitivity is untouched by a full sync and by bare Field updates"
+ (testing "a user-set data_sensitivity lives in FieldUserSettings, untouched by a full sync or a bare Field update"
(mt/with-temp-test-data [["sens_table"
[{:field-name "email", :base-type :type/Text}]
@@ -605,17 +615,17 @@
(is (nil? (field)))
(is (not (t2/exists? :model/FieldUserSettings :field_id field-id))))
(mt/user-http-request :crowberto :put 200 (format "field/%d" field-id) {:data_sensitivity "PII"})
- (testing "after a full sync"
+ (testing "after a full sync, the classifier skips the user-overridden Field"
(sync/sync-database! db)
- (is (= :PII (field)))
+ (is (nil? (field)))
(is (= :PII (mirror))))
- (testing "after a bare update to the Field with a different label"
+ (testing "a bare update to the Field only changes the classifier's own value"
(t2/update! :model/Field field-id {:data_sensitivity :SEC_KEY})
- (is (= :PII (field)))
+ (is (= :SEC_KEY (field)))
(is (= :PII (mirror))))))))
(deftest data-sensitivity-classifier-respects-user-label-test
- (testing "a user-set data_sensitivity wins over the classifier's inference while sibling fields get labeled"
+ (testing "a user-set data_sensitivity in FieldUserSettings survives the classifier reclassifying the raw Field"
(mt/with-temp-test-data [["app_users"
[{:field-name "email", :base-type :type/Text}
{:field-name "ssn", :base-type :type/Text}
@@ -627,8 +637,9 @@
(is (nil? (field :ssn)))
(mt/with-temporary-setting-values [data-sensitivity-scan-enabled true]
(sync/sync-database! (mt/db)))
- (testing "the user's PUBLIC on email survives even though the rules say PII"
- (is (= :PUBLIC (field :email)))
+ (testing "the classifier still infers and writes its own category on the raw Field"
+ (is (= :PII (field :email))))
+ (testing "the user's PUBLIC override on email survives in FieldUserSettings"
(is (= :PUBLIC (mirror :email))))
(testing "the classifier labels the unlabeled siblings without creating mirror rows"
(is (= :PII (field :ssn)))
@@ -645,16 +656,18 @@
(let [db (mt/db)
db-spec (sql-jdbc.conn/db->pooled-connection-spec db)
field-id (mt/id :readd_table :email)
- field #(t2/select-one [:model/Field :active :data_sensitivity] :id field-id)]
+ field #(t2/select-one [:model/Field :active :data_sensitivity] :id field-id)
+ mirror #(t2/select-one-fn :data_sensitivity :model/FieldUserSettings :field_id field-id)]
(mt/user-http-request :crowberto :put 200 (format "field/%d" field-id) {:data_sensitivity "PHI"})
(jdbc/execute! db-spec ["ALTER TABLE \"READD_TABLE\" DROP COLUMN \"EMAIL\";"])
(sync/sync-database! db)
- (testing "the field is inactive while the column is gone"
- (is (=? {:active false, :data_sensitivity :PHI} (field))))
+ (testing "the field is inactive while the column is gone; the mirror keeps the label"
+ (is (=? {:active false, :data_sensitivity nil} (field)))
+ (is (= :PHI (mirror))))
(jdbc/execute! db-spec ["ALTER TABLE \"READD_TABLE\" ADD COLUMN \"EMAIL\" VARCHAR;"])
(sync/sync-database! db)
- (testing "the same Field row is reactivated and keeps its label"
- (is (=? {:active true, :data_sensitivity :PHI} (field)))
- (is (= :PHI (t2/select-one-fn :data_sensitivity :model/FieldUserSettings :field_id field-id)))))
+ (testing "the same Field row is reactivated and the mirror still keeps its label"
+ (is (=? {:active true, :data_sensitivity nil} (field)))
+ (is (= :PHI (mirror)))))
(finally
(t2/delete! :model/Database (mt/id)))))))
diff --git a/test/metabase/test/data/impl/get_or_create.clj b/test/metabase/test/data/impl/get_or_create.clj
index 2b11bbad9e7..05636a567fe 100644
--- a/test/metabase/test/data/impl/get_or_create.clj
+++ b/test/metabase/test/data/impl/get_or_create.clj
@@ -6,7 +6,6 @@
[medley.core :as m]
[metabase.api.common :as api]
[metabase.driver :as driver]
- [metabase.models.humanization :as humanization]
[metabase.permissions.models.data-permissions :as data-perms]
[metabase.permissions.models.permissions-group :as perms-group]
[metabase.sync.analyze :as sync.analyze]
@@ -18,6 +17,8 @@
[metabase.util :as u]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
+ [metabase.warehouse-schema.humanization :as warehouse-schema.humanization]
+ [metabase.warehouse-schema.settings :as warehouse-schema.settings]
[toucan2.core :as t2]
[toucan2.tools.with-temp])
(:import
@@ -177,7 +178,7 @@
(some? fk) :type/FK
:else semantic-type)]
{:name field-name
- :display_name (humanization/name->human-readable-name :simple field-name)
+ :display_name (warehouse-schema.humanization/name->human-readable-name :simple field-name)
:database_type database-type
:base_type actual-base-type
:effective_type (or effective-type actual-base-type)
@@ -199,7 +200,7 @@
table-row {:db_id db-id
:name sync-table-name
:schema schema
- :display_name (humanization/name->human-readable-name :simple table-name)
+ :display_name (warehouse-schema.humanization/name->human-readable-name :simple table-name)
:description table-comment
:active true
:visibility_type nil
@@ -281,7 +282,7 @@
;;; ----------------------------------------------- End Fake Sync -----------------------------------------------
(defn- sync-newly-created-database! [driver {:keys [database-name], :as database-definition} connection-details db]
- (assert (= (humanization/humanization-strategy) :simple)
+ (assert (= (warehouse-schema.settings/humanization-strategy) :simple)
"Humanization strategy is not set to the default value of :simple! Metadata will be broken!")
(try
(u/with-timeout sync-timeout-ms
diff --git a/test/metabase/upload/impl_test.clj b/test/metabase/upload/impl_test.clj
index e6d6dbbad9e..ac3a4127310 100644
--- a/test/metabase/upload/impl_test.clj
+++ b/test/metabase/upload/impl_test.clj
@@ -35,6 +35,7 @@
[metabase.upload.types :as upload-types]
[metabase.util :as u]
[metabase.util.malli :as mu]
+ [metabase.warehouse-schema.models.field-user-settings :as field-user-settings]
[toucan2.core :as t2])
(:import
(java.io ByteArrayInputStream File FileOutputStream)))
@@ -2528,8 +2529,9 @@
(is (= (header-with-auto-pk ["A"])
(column-display-names-for-table table))))
(testing "But we can configure it"
- (t2/update! :model/Field {:name "a" :table_id (:id table)}
- {:display_name bespoke-name})
+ (field-user-settings/upsert-user-settings
+ {:id (t2/select-one-pk :model/Field :name "a" :table_id (:id table))}
+ {:display_name bespoke-name})
(is (= (header-with-auto-pk [bespoke-name])
(column-display-names-for-table table))))
(let [file (csv-file-with data (mt/random-name))]
diff --git a/test/metabase/warehouse_schema/models/field_test.clj b/test/metabase/warehouse_schema/models/field_test.clj
index a72faad0c93..fd24e3fbeea 100644
--- a/test/metabase/warehouse_schema/models/field_test.clj
+++ b/test/metabase/warehouse_schema/models/field_test.clj
@@ -7,6 +7,8 @@
[metabase.permissions.models.data-permissions :as data-perms]
[metabase.test :as mt]
[metabase.util :as u]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
+ [metabase.warehouse-schema.db :as warehouse-schema.db]
[metabase.warehouse-schema.models.field :as field]
[metabase.warehouse-schema.models.field-user-settings :as field-user-settings]
[toucan2.core :as t2]))
@@ -284,25 +286,6 @@
:database_position 0}))]
(t2/update! :model/Field id {:effective_type :type/Text
:coercion_strategy nil})
- id))]
- ["upsert-user-settings writes broken effective_type then any field update fires the merge-back overlay"
- (fn [table-id]
- (let [id (first (t2/insert-returning-pks!
- :model/Field
- {:table_id table-id
- :name "upsert_then_upd"
- :display_name "upsert_then_upd"
- :database_type "NUMBER"
- :base_type :type/Number
- :effective_type :type/Number
- :position 0
- :database_position 0}))]
- ;; write a stale, no-coercion overlay into user-settings
- (field-user-settings/upsert-user-settings
- {:id id}
- {:effective_type :type/Text :coercion_strategy nil})
- ;; trigger before-update (which runs sync-user-settings overlay merge)
- (t2/update! :model/Field id {:display_name "trigger merge"})
id))]]]
(testing label
(mt/with-temp [:model/Database {db-id :id} {}
@@ -320,27 +303,107 @@
(is (nil? (t2/select-one-fn :data_sensitivity :model/Field :id field-id)))
(is (not (t2/exists? :model/FieldUserSettings :field_id field-id))))))
-(deftest data-sensitivity-user-setting-overlay-test
- (testing "a user-set label wins over a bare update to the Field"
+(deftest data-sensitivity-user-setting-test
+ (testing "a user-set label lives in the user settings and does not touch the Field"
(mt/with-temp [:model/Field {field-id :id :as field} {:data_sensitivity :PII}]
(field-user-settings/upsert-user-settings field {:data_sensitivity :PUBLIC})
- (t2/update! :model/Field field-id {:data_sensitivity :PII})
- (is (= :PUBLIC (t2/select-one-fn :data_sensitivity :model/Field :id field-id)))
- (is (= :PUBLIC (t2/select-one-fn :data_sensitivity :model/FieldUserSettings :field_id field-id)))))
- (testing "a user-set label also overlays an update that does not mention data_sensitivity"
+ (is (= :PII (t2/select-one-fn :data_sensitivity :model/Field :id field-id)))
+ (is (= :PUBLIC (t2/select-one-fn :data_sensitivity :model/FieldUserSettings :field_id field-id)))
+ (is (= :PUBLIC (:data_sensitivity (warehouse-schema.db/field field-id))))))
+ (testing "a bare update to the Field is written as given and stays hidden behind the user label"
(mt/with-temp [:model/Field {field-id :id :as field} {:data_sensitivity :PII}]
(field-user-settings/upsert-user-settings field {:data_sensitivity :PUBLIC})
- (t2/update! :model/Field field-id {:description "touched"})
- (is (= :PUBLIC (t2/select-one-fn :data_sensitivity :model/Field :id field-id)))))
- (testing "a nil user setting does not block a bare update to the Field"
- (mt/with-temp [:model/Field {field-id :id :as field}]
- (field-user-settings/upsert-user-settings field {:data_sensitivity nil})
- (is (t2/exists? :model/FieldUserSettings :field_id field-id))
- (t2/update! :model/Field field-id {:data_sensitivity :PII})
- (is (= :PII (t2/select-one-fn :data_sensitivity :model/Field :id field-id)))
- (is (nil? (t2/select-one-fn :data_sensitivity :model/FieldUserSettings :field_id field-id)))))
+ (t2/update! :model/Field field-id {:data_sensitivity :PHI})
+ (is (= :PHI (t2/select-one-fn :data_sensitivity :model/Field :id field-id)))
+ (is (= :PUBLIC (:data_sensitivity (warehouse-schema.db/field field-id))))))
(testing "upsert-user-settings with a nil value clears a previously set label on the mirror"
(mt/with-temp [:model/Field {field-id :id :as field} {:data_sensitivity :PII}]
(field-user-settings/upsert-user-settings field {:data_sensitivity :PUBLIC})
(field-user-settings/upsert-user-settings field {:data_sensitivity nil})
- (is (nil? (t2/select-one-fn :data_sensitivity :model/FieldUserSettings :field_id field-id))))))
+ (is (nil? (t2/select-one-fn :data_sensitivity :model/FieldUserSettings :field_id field-id)))
+ (is (= :PII (:data_sensitivity (warehouse-schema.db/field field-id)))))))
+
+(deftest field-user-settings-read-test
+ (mt/with-temp [:model/Field {edited-id :id :as edited} {:display_name "Sync Name"
+ :description "sync description"
+ :semantic_type :type/Category
+ :base_type :type/Text
+ :effective_type :type/Text
+ :coercion_strategy nil}
+ :model/Field {plain-id :id} {:display_name "Plain"}]
+ (field-user-settings/upsert-user-settings edited {:display_name "User Name" :description nil :semantic_type nil})
+ (testing "the sync row is untouched"
+ (is (=? {:display_name "Sync Name" :description "sync description" :semantic_type :type/Category}
+ (t2/select-one :model/Field :id edited-id))))
+ (testing "user values, a user NULL included, replace the sync ones; the rest of the row and its transforms are intact"
+ (is (=? {:id edited-id :display_name "User Name" :description nil :semantic_type nil :base_type :type/Text
+ :effective_type :type/Text :coercion_strategy nil :visibility_type :normal}
+ (warehouse-schema.db/field edited-id)))
+ (is (=? {:id plain-id :display_name "Plain"} (warehouse-schema.db/field plain-id))))
+ (testing "a flag left false means the sync value shows, even next to a user value"
+ (t2/update! :model/FieldUserSettings edited-id {:description_set false})
+ (is (= "sync description" (:description (warehouse-schema.db/field edited-id)))))
+ (testing "coercion_strategy follows the user's effective_type, so a cleared coercion shows as cleared"
+ (t2/update! :model/Field edited-id {:effective_type :type/Number :coercion_strategy :Coercion/String->Number})
+ (field-user-settings/upsert-user-settings edited {:effective_type :type/Text :coercion_strategy nil})
+ (is (=? {:effective_type :type/Text :coercion_strategy nil} (warehouse-schema.db/field edited-id)))
+ (field-user-settings/unset-user-settings! edited [:effective_type :coercion_strategy])
+ (is (=? {:effective_type :type/Number :coercion_strategy :Coercion/String->Number} (warehouse-schema.db/field edited-id))))))
+
+(deftest upsert-user-settings-flags-test
+ (mt/with-temp [:model/Field {field-id :id :as field} {}
+ :model/Field {target-id :id} {}]
+ (testing "only the flagged columns present in the settings are flagged"
+ (field-user-settings/upsert-user-settings field {:display_name "X" :semantic_type :type/FK :fk_target_field_id target-id})
+ (is (=? {:display_name "X" :semantic_type :type/FK :fk_target_field_id target-id
+ :description_set false :semantic_type_set true :fk_target_field_id_set true}
+ (t2/select-one :model/FieldUserSettings :field_id field-id))))
+ (testing "an explicit nil is recorded and flagged as set"
+ (field-user-settings/upsert-user-settings field {:description nil})
+ (is (=? {:description nil :description_set true :semantic_type_set true}
+ (t2/select-one :model/FieldUserSettings :field_id field-id))))
+ (testing "unset-user-settings! drops the values and their flags"
+ (field-user-settings/unset-user-settings! field [:semantic_type :fk_target_field_id :display_name])
+ (is (=? {:display_name nil :semantic_type nil :fk_target_field_id nil
+ :description_set true :semantic_type_set false :fk_target_field_id_set false}
+ (t2/select-one :model/FieldUserSettings :field_id field-id))))
+ (testing "dropping the last value deletes the row"
+ (field-user-settings/unset-user-settings! field [:description])
+ (is (nil? (t2/select-one :model/FieldUserSettings :field_id field-id))))
+ (testing "an upsert that leaves no value behind creates no row"
+ (field-user-settings/upsert-user-settings field {:display_name nil})
+ (is (nil? (t2/select-one :model/FieldUserSettings :field_id field-id))))
+ (testing "keys outside the user-settable set are ignored"
+ (field-user-settings/upsert-user-settings field {:name "nope" :position 3})
+ (is (nil? (:name (t2/select-one :model/FieldUserSettings :field_id field-id)))))))
+
+(deftest field-query-test
+ (testing "a query sourced from field-query reads the user's values, a user NULL included"
+ (mt/with-temp [:model/Field {field-id :id :as field} {:display_name "Sync" :description "sync" :semantic_type :type/Category}]
+ (field-user-settings/upsert-user-settings field {:display_name "User" :description nil})
+ (is (=? {:display_name "User" :description nil :semantic_type :type/Category}
+ (t2/select-one :model/Field :id field-id {:from [(warehouse-schema-overlay/field-query)]})))
+ (testing "and metabase_field itself still holds sync's values"
+ (is (=? {:display_name "Sync" :description "sync" :semantic_type :type/Category}
+ (t2/select-one :model/Field :id field-id))))))
+ (testing "{:user-settings? false} asks for sync's values from the same helper"
+ (mt/with-temp [:model/Field {field-id :id :as field} {:display_name "Sync"}]
+ (field-user-settings/upsert-user-settings field {:display_name "User"})
+ (is (= "Sync" (:display_name (t2/select-one :model/Field :id field-id
+ {:from [(warehouse-schema-overlay/field-query {:user-settings? false})]}))))))
+ (testing "a coercion the user cleared reads as cleared, following their effective_type"
+ (mt/with-temp [:model/Field {field-id :id :as field} {:base_type :type/Text
+ :effective_type :type/Number
+ :coercion_strategy :Coercion/String->Number}]
+ (field-user-settings/upsert-user-settings field {:effective_type :type/Text :coercion_strategy nil})
+ (is (=? {:effective_type :type/Text :coercion_strategy nil}
+ (t2/select-one :model/Field :id field-id {:from [(warehouse-schema-overlay/field-query)]}))))))
+
+(deftest deactivating-fk-target-unsets-user-fk-test
+ (testing "retiring a Field drops the user-set FKs pointing at it so the sync values show again"
+ (mt/with-temp [:model/Field {target-id :id} {}
+ :model/Field {source-id :id :as source} {:semantic_type :type/Category}]
+ (field-user-settings/upsert-user-settings source {:semantic_type :type/FK :fk_target_field_id target-id})
+ (t2/update! :model/Field target-id {:active false})
+ (is (nil? (t2/select-one :model/FieldUserSettings :field_id source-id)))
+ (is (= :type/Category (:semantic_type (warehouse-schema.db/field source-id)))))))
diff --git a/test/metabase/warehouse_schema/models/table_test.clj b/test/metabase/warehouse_schema/models/table_test.clj
index 7a5777cf7fa..5cbda33a6f6 100644
--- a/test/metabase/warehouse_schema/models/table_test.clj
+++ b/test/metabase/warehouse_schema/models/table_test.clj
@@ -14,35 +14,35 @@
[metabase.test.fixtures :as fixtures]
[metabase.util :as u]
[metabase.warehouse-schema.models.field-values :as field-values]
- [metabase.warehouse-schema.models.table :as table]
+ [metabase.warehouse-schema.models.table-user-settings :as table-user-settings]
[toucan2.core :as t2]))
(use-fixtures :once (fixtures/initialize :db :test-users))
(deftest valid-field-order?-test
(testing "A valid field ordering is a set IDs of all active fields in a given table"
- (is (#'table/valid-field-order? (mt/id :venues)
- [(mt/id :venues :name)
- (mt/id :venues :category_id)
- (mt/id :venues :latitude)
- (mt/id :venues :longitude)
- (mt/id :venues :price)
- (mt/id :venues :id)])))
+ (is (#'table-user-settings/valid-field-order? (mt/id :venues)
+ [(mt/id :venues :name)
+ (mt/id :venues :category_id)
+ (mt/id :venues :latitude)
+ (mt/id :venues :longitude)
+ (mt/id :venues :price)
+ (mt/id :venues :id)])))
(testing "Field ordering is invalid if some fields are missing"
- (is (false? (#'table/valid-field-order? (mt/id :venues)
- [(mt/id :venues :category_id)
- (mt/id :venues :latitude)
- (mt/id :venues :longitude)
- (mt/id :venues :price)
- (mt/id :venues :id)]))))
+ (is (false? (#'table-user-settings/valid-field-order? (mt/id :venues)
+ [(mt/id :venues :category_id)
+ (mt/id :venues :latitude)
+ (mt/id :venues :longitude)
+ (mt/id :venues :price)
+ (mt/id :venues :id)]))))
(testing "Field ordering is invalid if some fields are from a differnt table"
- (is (false? (#'table/valid-field-order? (mt/id :venues)
- [(mt/id :venues :name)
- (mt/id :venues :category_id)
- (mt/id :venues :latitude)
- (mt/id :venues :longitude)
- (mt/id :venues :price)
- (mt/id :checkins :id)]))))
+ (is (false? (#'table-user-settings/valid-field-order? (mt/id :venues)
+ [(mt/id :venues :name)
+ (mt/id :venues :category_id)
+ (mt/id :venues :latitude)
+ (mt/id :venues :longitude)
+ (mt/id :venues :price)
+ (mt/id :checkins :id)]))))
(testing "Only active fields should be considerd when checking field order"
(one-off-dbs/with-blank-db
(doseq [statement [;; H2 needs that 'guest' user for QP purposes. Set that up
@@ -59,13 +59,13 @@
"('Chicken', 'Colin Fowl');")]]
(jdbc/execute! one-off-dbs/*conn* [statement]))
(sync/sync-database! (mt/db))
- (is (#'table/valid-field-order? (mt/id :birds)
- [(mt/id :birds :species)
- (mt/id :birds :example_name)]))
+ (is (#'table-user-settings/valid-field-order? (mt/id :birds)
+ [(mt/id :birds :species)
+ (mt/id :birds :example_name)]))
(jdbc/execute! one-off-dbs/*conn* ["ALTER TABLE \"BIRDS\" DROP COLUMN \"EXAMPLE_NAME\";"])
(sync/sync-database! (mt/db))
- (is (#'table/valid-field-order? (mt/id :birds)
- [(mt/id :birds :species)])))))
+ (is (#'table-user-settings/valid-field-order? (mt/id :birds)
+ [(mt/id :birds :species)])))))
(deftest slashes-in-schema-names-test
(testing "Schema names should allow forward or back slashes (#8693, #12450)"
diff --git a/test/metabase/warehouse_schema/models/table_user_settings_test.clj b/test/metabase/warehouse_schema/models/table_user_settings_test.clj
new file mode 100644
index 00000000000..6f5ff7e47f9
--- /dev/null
+++ b/test/metabase/warehouse_schema/models/table_user_settings_test.clj
@@ -0,0 +1,104 @@
+(ns metabase.warehouse-schema.models.table-user-settings-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.test :as mt]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
+ [metabase.warehouse-schema.db :as warehouse-schema.db]
+ [metabase.warehouse-schema.models.table-user-settings :as table-user-settings]
+ [toucan2.core :as t2]))
+
+(defn- user-table
+ "The Table as users see it: what they set, else what sync wrote."
+ [table-id]
+ (t2/select-one :model/Table :id table-id {:from [(warehouse-schema-overlay/table-query)]}))
+
+(deftest table-query-test
+ (testing "a query sourced from table-query reads the user's values over sync's"
+ (mt/with-temp [:model/Table {table-id :id :as table} {:display_name "Sync Name"
+ :description "sync description"
+ :entity_type :entity/GenericTable}]
+ (table-user-settings/upsert-user-settings table {:display_name "User Name" :description nil})
+ (testing "the sync row is untouched"
+ (is (=? {:display_name "Sync Name" :description "sync description"}
+ (t2/select-one :model/Table :id table-id))))
+ (testing "user values, a user NULL included, replace the sync ones"
+ (is (=? {:id table-id :display_name "User Name" :description nil :entity_type :entity/GenericTable}
+ (user-table table-id))))
+ (testing "a flag left false means the sync value shows again"
+ (t2/update! :model/TableUserSettings table-id {:description_set false})
+ (is (= "sync description" (:description (user-table table-id))))))))
+
+(deftest not-null-columns-need-no-flag-test
+ (testing "a NULL in a column that is NOT NULL on the Table can only mean the user set nothing"
+ (mt/with-temp [:model/Table {table-id :id :as table} {:field_order :database}]
+ (is (= :database (:field_order (user-table table-id))))
+ (table-user-settings/upsert-user-settings table {:field_order :alphabetical})
+ (is (= :alphabetical (:field_order (user-table table-id))))
+ (testing "clearing it falls back to the Table's own value, and drops the row it emptied"
+ (table-user-settings/unset-user-settings! table [:field_order])
+ (is (= :database (:field_order (user-table table-id))))
+ (is (nil? (t2/select-one :model/TableUserSettings :table_id table-id)))))))
+
+(deftest visibility-pair-is-recorded-whole-test
+ (testing "visibility_type and data_layer are one choice, so setting either records both"
+ (mt/with-temp [:model/Table {table-id :id :as table} {}]
+ (table-user-settings/upsert-user-settings table {:visibility_type :hidden})
+ (is (=? {:visibility_type :hidden :data_layer :hidden} (user-table table-id)))
+ (table-user-settings/upsert-user-settings table {:data_layer :internal})
+ (is (=? {:visibility_type nil :data_layer :internal} (user-table table-id))))))
+
+(deftest publishing-is-recorded-as-a-pair-test
+ (testing "collection_id is only meaningful alongside is_published, which is NOT NULL and answers for the pair"
+ (mt/with-temp [:model/Collection {collection-id :id} {}
+ :model/Table {table-id :id :as table} {}]
+ (table-user-settings/upsert-user-settings table {:is_published true :collection_id collection-id})
+ (is (=? {:is_published true :collection_id collection-id} (user-table table-id)))
+ (testing "a Table published into the root collection keeps its NULL collection_id"
+ (table-user-settings/upsert-user-settings table {:is_published true :collection_id nil})
+ (is (=? {:is_published true :collection_id nil} (user-table table-id)))))))
+
+(deftest bulk-upsert-test
+ (testing "a bulk edit records the same values on every Table it names"
+ (mt/with-temp [:model/Table {t1 :id} {}
+ :model/Table {t2 :id} {}]
+ (table-user-settings/upsert-user-settings-for-tables! #{t1 t2} {:owner_email "[email protected]"})
+ (map (comp :owner_email user-table) [t1 t2])))
+ (is (= 2 (count (warehouse-schema.db/table-ids-with-user-settings #{t1 t2})))))))
+
+(deftest invariants-hold-however-the-row-is-written-test
+ (testing "the model's hooks, not just the upsert fns, enforce what has to hold of a settings row -- so a serdes
+ import or any other writer cannot store one that readers would misread"
+ (mt/with-temp [:model/Database {db-id :id} {}
+ :model/Table {table-id :id} {:db_id db-id}
+ :model/Field {field-id :id} {:table_id table-id}]
+ (testing "a written column carries its _set flag, on insert and on update"
+ (t2/insert! :model/FieldUserSettings {:field_id field-id :description "d"})
+ (is (true? (t2/select-one-fn :description_set :model/FieldUserSettings :field_id field-id)))
+ (t2/update! :model/FieldUserSettings field-id {:semantic_type :type/Category})
+ (is (true? (t2/select-one-fn :semantic_type_set :model/FieldUserSettings :field_id field-id))))
+ (testing "a flag the writer set itself is left alone, which is how a value is taken back"
+ (t2/insert! :model/TableUserSettings {:table_id table-id :display_name "n" :description "d"})
+ (t2/update! :model/TableUserSettings table-id {:description nil :description_set false})
+ (is (false? (t2/select-one-fn :description_set :model/TableUserSettings :table_id table-id))))
+ (testing "both halves of the visibility choice move together"
+ (t2/update! :model/TableUserSettings table-id {:visibility_type :hidden})
+ (is (=? {:visibility_type :hidden :data_layer :hidden
+ :visibility_type_set true :data_layer_set true}
+ (t2/select-one :model/TableUserSettings :table_id table-id))))
+ (testing "a change a user may not make is refused whoever writes it"
+ (t2/update! :model/TableUserSettings table-id {:data_authority :authoritative})
+ (is (thrown-with-msg? Exception #"Cannot set data_authority back to unconfigured"
+ (t2/update! :model/TableUserSettings table-id
+ {:data_authority :unconfigured})))))))
+
+(deftest custom-order-fields-test
+ (testing "a custom order is recorded as the Fields' user custom_position and applied to their position"
+ (mt/with-temp [:model/Table {table-id :id :as table} {}
+ :model/Field {f1 :id} {:table_id table-id :name "A"}
+ :model/Field {f2 :id} {:table_id table-id :name "B"}]
+ (table-user-settings/custom-order-fields! table [f2 f1])
+ (is (= :custom (:field_order (user-table table-id))))
+ (is (= {f2 0 f1 1} (t2/select-fn->fn :field_id :custom_position :model/FieldUserSettings :field_id [:in [f1 f2]])))
+ (is (= [f2 f1] (t2/select-pks-vec :model/Field :table_id table-id {:order-by [[:position :asc]]})))
+ (is (= #{0} (t2/select-fn-set :custom_position :model/Field :table_id table-id))))))
diff --git a/test/metabase/models/humanization_test.clj b/test/metabase/warehouse_schema/settings_test.clj
similarity index 59%
rename from test/metabase/models/humanization_test.clj
rename to test/metabase/warehouse_schema/settings_test.clj
index 9c544c476e8..56f74481400 100644
--- a/test/metabase/models/humanization_test.clj
+++ b/test/metabase/warehouse_schema/settings_test.clj
@@ -1,13 +1,16 @@
-(ns metabase.models.humanization-test
+(ns metabase.warehouse-schema.settings-test
(:require
[clojure.test :refer :all]
- [metabase.models.humanization :as humanization]
[metabase.test :as mt]
[metabase.test.util :as tu]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
+ [metabase.warehouse-schema.humanization :as humanization]
+ [metabase.warehouse-schema.models.field-user-settings :as field-user-settings]
+ [metabase.warehouse-schema.settings :as warehouse-schema.settings]
[toucan2.core :as t2]))
(defn- get-humanized-display-name! [actual-name strategy]
- (mt/with-dynamic-fn-redefs [humanization/humanization-strategy (constantly strategy)]
+ (tu/with-temporary-setting-values [humanization-strategy strategy]
(mt/with-temp [:model/Table {table-id :id} {:name actual-name}]
(t2/select-one-fn :display_name :model/Table, :id table-id))))
@@ -37,11 +40,11 @@
(is (= (:initial expected)
(display-name))))
(testing "switch to :simple"
- (humanization/humanization-strategy! "simple")
+ (warehouse-schema.settings/humanization-strategy! "simple")
(is (= (:simple expected)
(display-name))))
(testing "switch to :none"
- (humanization/humanization-strategy! "none")
+ (warehouse-schema.settings/humanization-strategy! "none")
(is (= (:none expected)
(display-name))))))))))
@@ -52,11 +55,28 @@
(mt/with-temp [:model/Table {table-id :id} {:name "toucansare_cool", :display_name "My Favorite Table"}]
(doseq [new-strategy ["simple" "none"]]
(testing (format "switch from %s -> %s" initial-strategy new-strategy)
- (humanization/humanization-strategy! new-strategy)
+ (warehouse-schema.settings/humanization-strategy! new-strategy)
(is (= "My Favorite Table"
(t2/select-one-fn :display_name :model/Table, :id table-id))))))))))
+(deftest do-not-overwrite-user-set-field-display-names-test
+ (testing "a Field's display name is custom when it's set in FieldUserSettings or differs from the old strategy's humanization"
+ (tu/with-temporary-setting-values [humanization-strategy "simple"]
+ (mt/with-temp [:model/Field {user-set-id :id} {:name "toucansare_cool", :display_name "Toucansare Cool"}
+ :model/Field {custom-id :id} {:name "fussybird_sightings", :display_name "Some Other Name"}
+ :model/Field {synced-id :id} {:name "bird_watchers", :display_name "Bird Watchers"}]
+ (field-user-settings/upsert-user-settings {:id user-set-id} {:display_name "User's Name"})
+ (warehouse-schema.settings/humanization-strategy! "none")
+ (testing "the user-set Field's raw display name is left alone, and their own name is what reads back"
+ (is (= "Toucansare Cool" (t2/select-one-fn :display_name :model/Field, :id user-set-id)))
+ (is (= "User's Name" (t2/select-one-fn :display_name :model/Field, :id user-set-id
+ {:from [(warehouse-schema-overlay/field-query)]}))))
+ (testing "a raw display name that differs from the old humanization is left alone"
+ (is (= "Some Other Name" (t2/select-one-fn :display_name :model/Field, :id custom-id))))
+ (testing "a humanized display name is rewritten to the new strategy"
+ (is (= "bird_watchers" (t2/select-one-fn :display_name :model/Field, :id synced-id))))))))
+
(deftest invalid-strategies-default-to-simple
(tu/with-temporary-raw-setting-values [humanization-strategy "invalid-choice"]
- (is (= :simple (humanization/humanization-strategy)))
+ (is (= :simple (warehouse-schema.settings/humanization-strategy)))
(is (= "Foo Bar" (humanization/name->human-readable-name "foo_bar")))))
diff --git a/test/metabase/warehouse_schema_overlay/core_test.clj b/test/metabase/warehouse_schema_overlay/core_test.clj
new file mode 100644
index 00000000000..8d78a1477e6
--- /dev/null
+++ b/test/metabase/warehouse_schema_overlay/core_test.clj
@@ -0,0 +1,18 @@
+(ns metabase.warehouse-schema-overlay.core-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.util.malli :as mu]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
+ [metabase.warehouse-schema.schema]))
+
+(comment metabase.warehouse-schema.schema/keep-me)
+
+(deftest ^:parallel table-columns-match-the-table-schema-test
+ (testing "the spelled-out column list keeps up with the Table schema it cannot require"
+ (is (= (set (mu/map-schema-keys :metabase.warehouse-schema.schema/table))
+ warehouse-schema-overlay/table-columns))))
+
+(deftest ^:parallel field-columns-match-the-field-schema-test
+ (testing "the spelled-out column list keeps up with the Field schema it cannot require"
+ (is (= (set (mu/map-schema-keys :metabase.warehouse-schema.schema/field))
+ warehouse-schema-overlay/field-columns))))
diff --git a/test/metabase/warehouse_schema_rest/api/field_test.clj b/test/metabase/warehouse_schema_rest/api/field_test.clj
index 5620c2ecace..c6560b60406 100644
--- a/test/metabase/warehouse_schema_rest/api/field_test.clj
+++ b/test/metabase/warehouse_schema_rest/api/field_test.clj
@@ -109,6 +109,11 @@
["distincts" 75]]
(mt/user-http-request :crowberto :get 200 (format "field/%d/summary" (mt/id :categories :name)))))))
+(defn- sync-field
+ "The Field row as sync wrote it: what `metabase_field` itself holds, with no user values applied."
+ [field-id]
+ (t2/select-one :model/Field :id field-id))
+
(defn simple-field-details [field]
(select-keys field [:name
:display_name
@@ -129,7 +134,7 @@
(testing "PUT /api/field/:id"
(testing "test that we can do basic field update work, including unsetting some fields such as semantic-type"
(mt/with-temp [:model/Field {field-id :id} {:name "Field Test"}]
- (let [original-val (simple-field-details (t2/select-one :model/Field :id field-id))]
+ (let [original-field (simple-field-details (sync-field field-id))]
(testing "orignal value"
(is (= {:name "Field Test"
:display_name "Field Test"
@@ -140,7 +145,7 @@
:fk_target_field_id nil
:nfc_path nil
:data_sensitivity nil}
- original-val)))
+ original-field)))
(let [;; set it
response (mt/user-http-request :crowberto :put 200
(format "field/%d" field-id)
@@ -151,9 +156,8 @@
:json_unfolding true
:visibility_type :sensitive
:nfc_path ["bob" "dobbs"]
- :data_sensitivity "PII"})
- updated-val (simple-field-details (t2/select-one :model/Field :id field-id))]
- (testing "response body should be the updated field"
+ :data_sensitivity "PII"})]
+ (testing "response body shows the user's values"
(is (= {:name "Field Test"
:display_name "yay"
:description "foobar"
@@ -164,52 +168,81 @@
:nfc_path ["bob" "dobbs"]
:data_sensitivity "PII"}
(simple-field-details response))))
- (testing "updated value"
+ (testing ":model/Field stays at the sync value; :model/FieldUserSettings holds the user's values"
+ (is (= original-field (simple-field-details (sync-field field-id))))
+ (is (=? {:display_name "yay"
+ :description "foobar"
+ :description_set true
+ :semantic_type :type/Name
+ :semantic_type_set true
+ :visibility_type :sensitive
+ :json_unfolding true
+ :nfc_path ["bob" "dobbs"]
+ :data_sensitivity :PII}
+ (t2/select-one :model/FieldUserSettings :field_id field-id))))
+ (testing "GET /api/field/:id shows the user's values"
(is (= {:name "Field Test"
:display_name "yay"
:description "foobar"
- :semantic_type :type/Name
- :visibility_type :sensitive
+ :semantic_type "type/Name"
+ :visibility_type "sensitive"
:json_unfolding true
:fk_target_field_id nil
:nfc_path ["bob" "dobbs"]
- :data_sensitivity :PII}
- updated-val)))
+ :data_sensitivity "PII"}
+ (simple-field-details (mt/user-http-request :crowberto :get 200 (format "field/%d" field-id))))))
;; unset it
(mt/user-http-request :crowberto :put 200 (format "field/%d" field-id) {:description nil
:semantic_type nil
:nfc_path nil
:data_sensitivity nil})
- (testing "response"
+ (testing ":model/Field is still at the sync value after clearing the user's values"
+ (is (= original-field (simple-field-details (sync-field field-id)))))
+ (testing "GET shows the cleared values"
(is (= {:name "Field Test"
:display_name "yay"
:description nil
:semantic_type nil
- :visibility_type :sensitive
+ :visibility_type "sensitive"
:json_unfolding true
:fk_target_field_id nil
:nfc_path nil
:data_sensitivity nil}
- (simple-field-details (t2/select-one :model/Field :id field-id)))))))))))
+ (simple-field-details (mt/user-http-request :crowberto :get 200 (format "field/%d" field-id))))))))))))
+
+(deftest update-field-clear-description-test
+ (testing "PUT /api/field/:id with description nil clears the description users see"
+ (mt/with-temp [:model/Field {field-id :id} {:name "Field Test" :description "synced description"}]
+ (mt/user-http-request :crowberto :put 200 (format "field/%d" field-id) {:description "user description"})
+ (is (= "user description"
+ (:description (mt/user-http-request :crowberto :get 200 (format "field/%d" field-id)))))
+ (mt/user-http-request :crowberto :put 200 (format "field/%d" field-id) {:description nil})
+ (testing "the user's cleared description wins even though metabase_field.description is untouched"
+ (is (= "synced description" (:description (sync-field field-id))))
+ (is (nil? (:description (mt/user-http-request :crowberto :get 200 (format "field/%d" field-id)))))
+ (is (=? {:description nil :description_set true}
+ (t2/select-one :model/FieldUserSettings :field_id field-id)))))))
(deftest update-field-data-sensitivity-test
(testing "PUT /api/field/:id"
(mt/with-temp [:model/Field {field-id :id} {:name "Field Test"}]
- (testing "setting data_sensitivity writes the same value to the Field and to the user-settings mirror"
+ (testing "setting data_sensitivity writes only to :model/FieldUserSettings"
(is (= "SEC_KEY"
(:data_sensitivity (mt/user-http-request :crowberto :put 200 (format "field/%d" field-id)
{:data_sensitivity "SEC_KEY"}))))
- (is (= :SEC_KEY (t2/select-one-fn :data_sensitivity :model/Field :id field-id)))
+ (is (nil? (:data_sensitivity (sync-field field-id))))
(is (= :SEC_KEY (t2/select-one-fn :data_sensitivity :model/FieldUserSettings :field_id field-id))))
- (testing "a null data_sensitivity clears both tables"
+ (testing "a null data_sensitivity clears the user setting"
(is (nil? (:data_sensitivity (mt/user-http-request :crowberto :put 200 (format "field/%d" field-id)
{:data_sensitivity nil}))))
- (is (nil? (t2/select-one-fn :data_sensitivity :model/Field :id field-id)))
+ (is (nil? (:data_sensitivity (sync-field field-id))))
(is (nil? (t2/select-one-fn :data_sensitivity :model/FieldUserSettings :field_id field-id))))
(testing "an update that omits data_sensitivity leaves it unchanged"
(mt/user-http-request :crowberto :put 200 (format "field/%d" field-id) {:data_sensitivity "PHI"})
(mt/user-http-request :crowberto :put 200 (format "field/%d" field-id) {:description "unrelated"})
- (is (= :PHI (t2/select-one-fn :data_sensitivity :model/Field :id field-id)))))))
+ (is (nil? (:data_sensitivity (sync-field field-id))))
+ (is (= :PHI (t2/select-one-fn :data_sensitivity :model/FieldUserSettings :field_id field-id)))
+ (is (= "PHI" (:data_sensitivity (mt/user-http-request :crowberto :get 200 (format "field/%d" field-id)))))))))
(deftest update-field-data-sensitivity-validation-test
(testing "PUT /api/field/:id rejects data_sensitivity values outside the enum"
@@ -219,7 +252,7 @@
(is (=? {:errors {:data_sensitivity some?}}
(mt/user-http-request :crowberto :put 400 (format "field/%d" field-id)
{:data_sensitivity bad-value})))
- (is (nil? (t2/select-one-fn :data_sensitivity :model/Field :id field-id))))))))
+ (is (nil? (:data_sensitivity (sync-field field-id)))))))))
(deftest update-field-test-2
(testing "PUT /api/field/:id"
@@ -261,14 +294,35 @@
(mt/user-http-request :crowberto :put 200 (format "field/%d" field-id)
{:coercion_strategy strategy}))]
;; ensure that there is no coercion strategy from previous tests
- (set-strategy! nil)
- (let [field (t2/select-one :model/Field :id field-id)]
- (is (= :type/Integer (:effective_type field)))
- (is (contains? (get-in field [:fingerprint :type]) :type/Number)))
- (set-strategy! :Coercion/UNIXSeconds->DateTime)
- (let [field (t2/select-one :model/Field :id field-id)]
- (is (= :type/Instant (:effective_type field)))
- (is (contains? (get-in field [:fingerprint :type]) :type/DateTime))))))))))
+ (let [response (set-strategy! nil)]
+ (is (= "type/Integer" (:effective_type response)))
+ (is (contains? (get-in (t2/select-one :model/Field :id field-id) [:fingerprint :type]) :type/Number)))
+ (let [response (set-strategy! :Coercion/UNIXSeconds->DateTime)]
+ (is (= "type/Instant" (:effective_type response)))
+ (is (= "type/Instant" (:effective_type (mt/user-http-request :crowberto :get 200 (format "field/%d" field-id)))))
+ (is (not= :type/Instant (:effective_type (sync-field field-id))))
+ (is (contains? (get-in (t2/select-one :model/Field :id field-id) [:fingerprint :type]) :type/DateTime))))))))))
+
+(deftest update-field-coercion-set-then-cleared-test
+ (testing "PUT /api/field/:id"
+ (mt/with-temp [:model/Field {field-id :id} {:name "Field Test"}]
+ (testing "setting a coercion strategy is user-set only"
+ (let [response (mt/user-http-request :crowberto :put 200 (format "field/%d" field-id)
+ {:coercion_strategy :Coercion/YYYYMMDDHHMMSSString->Temporal})]
+ (is (= "Coercion/YYYYMMDDHHMMSSString->Temporal" (:coercion_strategy response)))
+ (is (= "type/DateTime" (:effective_type response))))
+ (is (nil? (:coercion_strategy (sync-field field-id))))
+ (is (= :Coercion/YYYYMMDDHHMMSSString->Temporal
+ (t2/select-one-fn :coercion_strategy :model/FieldUserSettings :field_id field-id)))
+ (is (= :type/DateTime
+ (t2/select-one-fn :effective_type :model/FieldUserSettings :field_id field-id))))
+ (testing "clearing it records the user's choice of no coercion, with the effective type back at the base type"
+ (let [response (mt/user-http-request :crowberto :put 200 (format "field/%d" field-id)
+ {:coercion_strategy nil})]
+ (is (nil? (:coercion_strategy response)))
+ (is (= "type/Text" (:effective_type response))))
+ (is (=? {:coercion_strategy nil :effective_type :type/Text}
+ (t2/select-one :model/FieldUserSettings :field_id field-id)))))))
(deftest update-field-test-3
(testing "PUT /api/field/:id"
@@ -276,7 +330,7 @@
(mt/with-temp [:model/Field {field-id :id} {:name "Field Test"}]
(mt/user-http-request :rasta :put 403 (format "field/%d" field-id) {:name "Field Test 2"})
(mt/user-http-request :rasta :put 403 (format "field/%d" field-id) {:data_sensitivity "PII"})
- (is (nil? (t2/select-one-fn :data_sensitivity :model/Field :id field-id)))))))
+ (is (nil? (:data_sensitivity (sync-field field-id))))))))
(deftest ^:parallel update-field-hydrated-target-test
(testing "PUT /api/field/:id"
@@ -289,18 +343,41 @@
(deftest remove-fk-semantic-type-test
(testing "PUT /api/field/:id"
- (testing "when we set the semantic-type from `:type/FK` to something else, make sure `:fk_target_field_id` is set to nil"
+ (testing "when we set the semantic-type from `:type/FK` to something else, `:fk_target_field_id` reads as nil
+ though :model/Field's is untouched"
(mt/with-temp [:model/Field {fk-field-id :id} {}
:model/Field {field-id :id} {:semantic_type :type/FK :fk_target_field_id fk-field-id}]
- (let [original-val (boolean (t2/select-one-fn :fk_target_field_id :model/Field, :id field-id))]
- (testing "before API call"
- (is (true?
- original-val)))
- ;; unset the :type/FK semantic-type
- (mt/user-http-request :crowberto :put 200 (format "field/%d" field-id) {:semantic_type :type/Name})
- (testing "after API call"
- (is (= nil
- (t2/select-one-fn :fk_target_field_id :model/Field, :id field-id)))))))))
+ (testing "before API call"
+ (is (= fk-field-id (:fk_target_field_id (sync-field field-id)))))
+ ;; unset the :type/FK semantic-type
+ (mt/user-http-request :crowberto :put 200 (format "field/%d" field-id) {:semantic_type :type/Name})
+ (testing "after API call"
+ (is (= fk-field-id (:fk_target_field_id (sync-field field-id))))
+ (is (=? {:semantic_type :type/Name
+ :semantic_type_set true
+ :fk_target_field_id nil}
+ (t2/select-one :model/FieldUserSettings :field_id field-id)))
+ (is (nil? (:fk_target_field_id (mt/user-http-request :crowberto :get 200 (format "field/%d" field-id))))))))))
+
+(deftest remove-sync-detected-fk-test
+ (testing "PUT /api/field/:id clearing semantic_type on a sync-detected FK hides the target without touching sync data"
+ (mt/with-temp [:model/Field {target-id :id} {}
+ :model/Field {field-id :id} {}]
+ (t2/update! :model/Field field-id {:semantic_type :type/FK :fk_target_field_id target-id})
+ (testing "before the user's edit, GET shows the sync-detected FK"
+ (is (=? {:semantic_type "type/FK"
+ :fk_target_field_id target-id
+ :target {:id target-id}}
+ (mt/user-http-request :crowberto :get 200 (format "field/%d" field-id)))))
+ (mt/user-http-request :crowberto :put 200 (format "field/%d" field-id) {:semantic_type nil})
+ (testing "after the user's edit, GET shows no FK or target"
+ (is (=? {:semantic_type nil
+ :fk_target_field_id nil
+ :target nil}
+ (mt/user-http-request :crowberto :get 200 (format "field/%d" field-id)))))
+ (testing ":model/Field still has the sync-detected FK"
+ (is (= :type/FK (:semantic_type (sync-field field-id))))
+ (is (= target-id (:fk_target_field_id (sync-field field-id))))))))
(deftest update-fk-target-field-id-test
(testing "PUT /api/field/:id"
@@ -308,8 +385,11 @@
(mt/with-temp [:model/Field {field-id :id} {:base_type :type/Integer}]
(mt/user-http-request :crowberto :put 200 (str "field/" field-id)
{:semantic_type :type/Quantity})
+ (is (nil? (:semantic_type (sync-field field-id))))
(is (= :type/Quantity
- (t2/select-one-fn :semantic_type :model/Field, :id field-id)))))))
+ (t2/select-one-fn :semantic_type :model/FieldUserSettings :field_id field-id)))
+ (is (= "type/Quantity"
+ (:semantic_type (mt/user-http-request :crowberto :get 200 (format "field/%d" field-id)))))))))
(defn- field->field-values
"Fetch the `FieldValues` object that corresponds to a given `Field`."
@@ -665,34 +745,22 @@
(mt/boolean-ids-and-timestamps (dimension-for-field field-id-1))))))))))
(deftest remove-fk-semantic-type-test-2
- (testing "When removing the FK semantic type, the fk_target_field_id should be cleared as well"
+ (testing "When removing the FK semantic type, the fk_target_field_id reads as cleared too, without touching :model/Field"
(mt/with-temp [:model/Field {field-id-1 :id} {:name "Field Test 1"}
:model/Field {field-id-2 :id} {:name "Field Test 2"
:semantic_type :type/FK
:fk_target_field_id field-id-1}]
- (testing "before change"
- (is (= {:name "Field Test 2"
- :display_name "Field Test 2"
- :description nil
- :visibility_type :normal
- :semantic_type :type/FK
- :fk_target_field_id true
- :json_unfolding false
- :nfc_path nil
- :data_sensitivity nil}
- (mt/boolean-ids-and-timestamps (simple-field-details (t2/select-one :model/Field :id field-id-2))))))
- (mt/user-http-request :crowberto :put 200 (format "field/%d" field-id-2) {:semantic_type nil})
- (testing "after change"
- (is (= {:name "Field Test 2"
- :display_name "Field Test 2"
- :description nil
- :visibility_type :normal
- :semantic_type nil
- :fk_target_field_id false
- :json_unfolding false
- :nfc_path nil
- :data_sensitivity nil}
- (mt/boolean-ids-and-timestamps (simple-field-details (t2/select-one :model/Field :id field-id-2)))))))))
+ (let [raw-field #(simple-field-details (sync-field field-id-2))
+ before (raw-field)]
+ (testing "before change"
+ (is (= {:semantic_type :type/FK, :fk_target_field_id field-id-1}
+ (select-keys before [:semantic_type :fk_target_field_id]))))
+ (mt/user-http-request :crowberto :put 200 (format "field/%d" field-id-2) {:semantic_type nil})
+ (testing "after change, :model/Field is untouched"
+ (is (= before (raw-field))))
+ (testing "after change, the user sees no FK"
+ (is (=? {:semantic_type nil, :fk_target_field_id nil}
+ (mt/user-http-request :crowberto :get 200 (format "field/%d" field-id-2)))))))))
(deftest update-fk-target-field-id-test-2
(testing "Checking update of the fk_target_field_id"
@@ -701,62 +769,33 @@
:model/Field {field-id-3 :id} {:name "Field Test 3"
:semantic_type :type/FK
:fk_target_field_id field-id-1}]
- (let [before-change (simple-field-details (t2/select-one :model/Field :id field-id-3))]
+ (let [raw-field #(simple-field-details (sync-field field-id-3))
+ before (raw-field)]
(testing "before change"
- (is (= {:name "Field Test 3"
- :display_name "Field Test 3"
- :description nil
- :visibility_type :normal
- :semantic_type :type/FK
- :fk_target_field_id true
- :json_unfolding false
- :nfc_path nil
- :data_sensitivity nil}
- (mt/boolean-ids-and-timestamps before-change))))
+ (is (= field-id-1 (:fk_target_field_id before))))
(mt/user-http-request :crowberto :put 200 (format "field/%d" field-id-3) {:fk_target_field_id field-id-2})
- (testing "after change"
- (let [after-change (simple-field-details (t2/select-one :model/Field :id field-id-3))]
- (is (= {:name "Field Test 3"
- :display_name "Field Test 3"
- :description nil
- :visibility_type :normal
- :semantic_type :type/FK
- :fk_target_field_id true
- :json_unfolding false
- :nfc_path nil
- :data_sensitivity nil}
- (mt/boolean-ids-and-timestamps after-change)))
- (is (not= (:fk_target_field_id before-change)
- (:fk_target_field_id after-change)))))))))
+ (testing "after change, :model/Field is untouched"
+ (is (= before (raw-field))))
+ (testing "after change, the user sees the new target"
+ (is (= field-id-2
+ (:fk_target_field_id (mt/user-http-request :crowberto :get 200 (format "field/%d" field-id-3))))))))))
(deftest update-fk-target-field-id-with-fk-test
(testing "Checking update of the fk_target_field_id along with an FK change"
(mt/with-temp [:model/Field {field-id-1 :id} {:name "Field Test 1"}
:model/Field {field-id-2 :id} {:name "Field Test 2"}]
- (testing "before change"
- (is (= {:name "Field Test 2"
- :display_name "Field Test 2"
- :description nil
- :visibility_type :normal
- :semantic_type nil
- :fk_target_field_id false
- :json_unfolding false
- :nfc_path nil
- :data_sensitivity nil}
- (mt/boolean-ids-and-timestamps (simple-field-details (t2/select-one :model/Field :id field-id-2))))))
- (mt/user-http-request :crowberto :put 200 (format "field/%d" field-id-2) {:semantic_type :type/FK
- :fk_target_field_id field-id-1})
- (testing "after change"
- (is (= {:name "Field Test 2"
- :display_name "Field Test 2"
- :description nil
- :visibility_type :normal
- :semantic_type :type/FK
- :fk_target_field_id true
- :json_unfolding false
- :nfc_path nil
- :data_sensitivity nil}
- (mt/boolean-ids-and-timestamps (simple-field-details (t2/select-one :model/Field :id field-id-2)))))))))
+ (let [raw-field #(simple-field-details (sync-field field-id-2))
+ before (raw-field)]
+ (testing "before change"
+ (is (= {:semantic_type nil, :fk_target_field_id nil}
+ (select-keys before [:semantic_type :fk_target_field_id]))))
+ (mt/user-http-request :crowberto :put 200 (format "field/%d" field-id-2) {:semantic_type :type/FK
+ :fk_target_field_id field-id-1})
+ (testing "after change, :model/Field is untouched"
+ (is (= before (raw-field))))
+ (testing "after change, the user sees the new FK"
+ (is (=? {:semantic_type "type/FK", :fk_target_field_id field-id-1}
+ (mt/user-http-request :crowberto :get 200 (format "field/%d" field-id-2)))))))))
(deftest fk-target-field-id-shouldnt-change-test
(testing "PUT /api/field/:id"
@@ -765,29 +804,17 @@
:model/Field {field-id-2 :id} {:name "Field Test 2"
:semantic_type :type/FK
:fk_target_field_id field-id-1}]
- (testing "before change"
- (is (= {:name "Field Test 2"
- :display_name "Field Test 2"
- :description nil
- :visibility_type :normal
- :semantic_type :type/FK
- :fk_target_field_id true
- :json_unfolding false
- :nfc_path nil
- :data_sensitivity nil}
- (mt/boolean-ids-and-timestamps (simple-field-details (t2/select-one :model/Field :id field-id-2))))))
- (mt/user-http-request :crowberto :put 200 (format "field/%d" field-id-2) {:description "foo"})
- (testing "after change"
- (is (= {:name "Field Test 2"
- :display_name "Field Test 2"
- :description "foo"
- :visibility_type :normal
- :semantic_type :type/FK
- :fk_target_field_id true
- :json_unfolding false
- :nfc_path nil
- :data_sensitivity nil}
- (mt/boolean-ids-and-timestamps (simple-field-details (t2/select-one :model/Field :id field-id-2))))))))))
+ (let [raw-field #(simple-field-details (sync-field field-id-2))
+ before (raw-field)]
+ (testing "before change"
+ (is (= {:semantic_type :type/FK, :fk_target_field_id field-id-1, :description nil}
+ (select-keys before [:semantic_type :fk_target_field_id :description]))))
+ (mt/user-http-request :crowberto :put 200 (format "field/%d" field-id-2) {:description "foo"})
+ (testing "after change, :model/Field is untouched, including its FK columns"
+ (is (= before (raw-field))))
+ (testing "after change, the user sees the new description and the same FK"
+ (is (=? {:semantic_type "type/FK", :fk_target_field_id field-id-1, :description "foo"}
+ (mt/user-http-request :crowberto :get 200 (format "field/%d" field-id-2))))))))))
(deftest update-field-type-dimension-test
(testing "PUT /api/field/:id"
@@ -988,15 +1015,17 @@
(mt/with-temp-copy-of-db
(mt/user-http-request :crowberto :put 200 (str "field/" (mt/id :venues :price))
{:coercion_strategy "Coercion/UNIXSeconds->DateTime"})
- (let [field (t2/select-one :model/Field :id (mt/id :venues :price))]
- (is (= :Coercion/UNIXSeconds->DateTime (:coercion_strategy field)))
- (is (isa? (:effective_type field) :type/DateTime)))
+ (let [field (mt/user-http-request :crowberto :get 200 (format "field/%d" (mt/id :venues :price)))]
+ (is (= "Coercion/UNIXSeconds->DateTime" (:coercion_strategy field)))
+ (is (isa? (keyword (:effective_type field)) :type/DateTime)))
(mt/user-http-request :crowberto :put 200 (str "field/" (mt/id :venues :price))
{:settings {:time_enabled "minutes"}})
- (let [field (t2/select-one :model/Field :id (mt/id :venues :price))]
- (is (= :Coercion/UNIXSeconds->DateTime (:coercion_strategy field)))
- (is (isa? (:effective_type field) :type/DateTime))
- (is (= "minutes" (-> field :settings :time_enabled)))))))
+ (let [field (mt/user-http-request :crowberto :get 200 (format "field/%d" (mt/id :venues :price)))]
+ (is (= "Coercion/UNIXSeconds->DateTime" (:coercion_strategy field)))
+ (is (isa? (keyword (:effective_type field)) :type/DateTime))
+ (is (= "minutes" (-> field :settings :time_enabled))))
+ (testing ":model/Field itself never gets a coercion strategy"
+ (is (nil? (:coercion_strategy (sync-field (mt/id :venues :price)))))))))
(deftest field-values-requires-query-permission-test
(testing "GET /api/field/:id/values requires query permission (view-data + create-queries)"
@@ -1023,7 +1052,9 @@
(testing (format "has_field_values = %s" v)
(mt/user-http-request :crowberto :put 200 (format "field/%d" fid) {:has_field_values v})
(is (= (keyword v)
- (t2/select-one-fn :has_field_values :model/Field :id fid))))))))
+ (t2/select-one-fn :has_field_values :model/FieldUserSettings :field_id fid)))
+ (is (= v
+ (:has_field_values (mt/user-http-request :crowberto :get 200 (format "field/%d" fid))))))))))
(deftest discard-field-values-test
(testing "POST /api/field/:id/discard_values"
diff --git a/test/metabase/warehouse_schema_rest/api/table_test.clj b/test/metabase/warehouse_schema_rest/api/table_test.clj
index 486b2eb91ec..d8d9b43af34 100644
--- a/test/metabase/warehouse_schema_rest/api/table_test.clj
+++ b/test/metabase/warehouse_schema_rest/api/table_test.clj
@@ -23,6 +23,7 @@
[metabase.upload.impl-test :as upload-test]
[metabase.util :as u]
[metabase.util.quick-task :as quick-task]
+ [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[metabase.warehouse-schema-rest.api.table :as api.table]
[toucan2.core :as t2])
(:import
@@ -34,6 +35,13 @@
;; We assume that all endpoints for a given context are enforced by the same middleware, so we don't run the same
;; authentication test on every single individual endpoint
+(defn- user-table-fn
+ "The value of `column` on the Table with `table-id` as users see it: what they set, else what sync wrote. A bare
+ `t2/select :model/Table` shows sync's values only."
+ [column table-id]
+ (t2/select-one-fn column :model/Table :id table-id
+ {:from [(warehouse-schema-overlay/table-query)]}))
+
(deftest ^:parallel unauthenticated-test
(is (= (get api.response/response-unauthentic :body)
(client/client :get 401 "table")))
@@ -495,7 +503,7 @@
(mt/with-temp [:model/Table table {:visibility_type "hidden"}]
(mt/user-http-request :crowberto :put 200 (format "table/%d" (u/the-id table))
{property (mt/random-name)})
- (is (= :hidden (t2/select-one-fn :visibility_type :model/Table :id (:id table)))))))))
+ (is (= :hidden (user-table-fn :visibility_type (:id table)))))))))
(deftest ^:parallel update-table-test-4
(testing "PUT /api/table/:id"
@@ -508,23 +516,23 @@
(testing "data_authority field behavior"
(mt/with-temp [:model/Table table {}]
(testing "Initially data_authority should be unconfigured"
- (is (= :unconfigured (t2/select-one-fn :data_authority :model/Table :id (u/the-id table)))))
+ (is (= :unconfigured (user-table-fn :data_authority (u/the-id table)))))
(testing "Can save an unrelated change with this field redundantly included"
(mt/user-http-request :crowberto :put 200 (format "table/%d" (u/the-id table))
{:active false, :data_authority "unconfigured"})
- (is (= :unconfigured (t2/select-one-fn :data_authority :model/Table :id (u/the-id table)))))
+ (is (= :unconfigured (user-table-fn :data_authority (u/the-id table)))))
(testing "Can set data_authority to authoritative"
(mt/user-http-request :crowberto :put 200 (format "table/%d" (u/the-id table))
{:data_authority "authoritative"})
- (is (= :authoritative (t2/select-one-fn :data_authority :model/Table :id (u/the-id table)))))
+ (is (= :authoritative (user-table-fn :data_authority (u/the-id table)))))
(testing "Can set data_authority between different values"
(mt/user-http-request :crowberto :put 200 (format "table/%d" (u/the-id table))
{:data_authority "computed"})
- (is (= :computed (t2/select-one-fn :data_authority :model/Table :id (u/the-id table)))))
+ (is (= :computed (user-table-fn :data_authority (u/the-id table)))))
(testing "Can set data_authority to ingested"
(mt/user-http-request :crowberto :put 200 (format "table/%d" (u/the-id table))
{:data_authority "ingested"})
- (is (= :ingested (t2/select-one-fn :data_authority :model/Table :id (u/the-id table)))))
+ (is (= :ingested (user-table-fn :data_authority (u/the-id table)))))
(testing "Cannot un-configure again"
(is (= "Cannot set data_authority back to unconfigured once it has been configured"
(mt/user-http-request :crowberto :put 400 (format "table/%d" (u/the-id table))
@@ -1095,7 +1103,7 @@
(mt/user-http-request :crowberto :post 404 (format "table/%d/discard_values" Integer/MAX_VALUE)))))))
(deftest field-ordering-test
- (let [original-field-order (t2/select-one-fn :field_order :model/Table :id (mt/id :venues))]
+ (let [original-field-order (user-table-fn :field_order (mt/id :venues))]
(try
(testing "Can we set alphabetical field ordering?"
(is (= ["CATEGORY_ID" "ID" "LATITUDE" "LONGITUDE" "NAME" "PRICE"]
@@ -1134,8 +1142,11 @@
(->> (t2/hydrate (t2/select-one :model/Table :id (mt/id :venues)) :fields)
:fields
(map u/the-id))))))
- (finally (mt/user-http-request :crowberto :put 200 (format "table/%s" (mt/id :venues))
- {:field_order original-field-order})))))
+ (finally
+ (mt/user-http-request :crowberto :put 200 (format "table/%s" (mt/id :venues))
+ {:field_order original-field-order})
+ (t2/delete! :model/TableUserSettings :table_id (mt/id :venues))
+ (t2/delete! :model/FieldUserSettings :field_id [:in (t2/select-pks-vec :model/Field :table_id (mt/id :venues))])))))
;;; +----------------------------------------------------------------------------------------------------------------+
;;; | POST /api/table/:id/append-csv |
@@ -1415,13 +1426,13 @@
(testing "updating visibility_type syncs to data_layer"
(mt/user-http-request :crowberto :put 200 (format "table/%d" (u/the-id table))
{:visibility_type "hidden"})
- (is (= :hidden (t2/select-one-fn :data_layer :model/Table :id (u/the-id table))))
- (is (= :hidden (t2/select-one-fn :visibility_type :model/Table :id (u/the-id table)))))
+ (is (= :hidden (user-table-fn :data_layer (u/the-id table))))
+ (is (= :hidden (user-table-fn :visibility_type (u/the-id table)))))
(testing "updating data_layer syncs to visibility_type"
(mt/user-http-request :crowberto :put 200 (format "table/%d" (u/the-id table))
{:data_layer "internal"})
- (is (= :internal (t2/select-one-fn :data_layer :model/Table :id (u/the-id table))))
- (is (= nil (t2/select-one-fn :visibility_type :model/Table :id (u/the-id table)))))
+ (is (= :internal (user-table-fn :data_layer (u/the-id table))))
+ (is (= nil (user-table-fn :visibility_type (u/the-id table)))))
(testing "cannot update both visibility_type and data_layer at once"
(is (= "Cannot update both visibility_type and data_layer"
(mt/user-http-request :crowberto :put 400 (format "table/%d" (u/the-id table))
@@ -1589,8 +1600,8 @@
(mt/user-http-request :crowberto :put 200 "table"
{:ids [t1 t2] :visibility_type "hidden"})
(is (= [:hidden :hidden]
- (map #(t2/select-one-fn :visibility_type :model/Table :id %) [t1 t2])))
+ (map #(user-table-fn :visibility_type %) [t1 t2])))
(mt/user-http-request :crowberto :put 200 "table"
{:ids [t1 t2] :visibility_type nil})
(is (= [nil nil]
- (map #(t2/select-one-fn :visibility_type :model/Table :id %) [t1 t2]))))))
+ (map #(user-table-fn :visibility_type %) [t1 t2]))))))
diff --git a/test/metabase/warehouses_rest/api_test.clj b/test/metabase/warehouses_rest/api_test.clj
index adcdc9eb6a7..8c2ab511c19 100644
--- a/test/metabase/warehouses_rest/api_test.clj
+++ b/test/metabase/warehouses_rest/api_test.clj
@@ -1028,6 +1028,23 @@
["CATEGORY_ID" "VENUES :type/Integer :type/FK"]]}]
(is (= expected (prefix-fn (mt/id) prefix)))))))
+(deftest autocomplete-suggestions-honors-user-set-semantic-type-and-visibility-test
+ (testing "GET /api/database/:id/autocomplete_suggestions honors the user's semantic_type/visibility_type"
+ (let [field-id (mt/id :venues :price)
+ base-type (t2/select-one-fn :base_type :model/Field field-id)
+ venues-row (fn [] (first (filter (fn [[_ desc]] (str/starts-with? desc "VENUES "))
+ (mt/user-http-request :rasta :get 200
+ (format "database/%d/autocomplete_suggestions" (mt/id))
+ :prefix "price"))))]
+ (testing "a user-set semantic_type replaces the sync value"
+ (mt/with-temp [:model/FieldUserSettings _ {:field_id field-id
+ :semantic_type :type/Currency
+ :semantic_type_set true}]
+ (is (= ["PRICE" (str "VENUES " base-type " :type/Currency")] (venues-row)))))
+ (testing "a user-set sensitive visibility_type hides the Field"
+ (mt/with-temp [:model/FieldUserSettings _ {:field_id field-id :visibility_type :sensitive}]
+ (is (nil? (venues-row))))))))
+
(deftest ^:parallel autocomplete-suggestions-test-2
(testing "GET /api/database/:id/autocomplete_suggestions"
(testing " returns sane Cache-Control headers"
diff --git a/test/metabase/xrays/automagic_dashboards/core_test.clj b/test/metabase/xrays/automagic_dashboards/core_test.clj
index 5caf13e68cf..897015cc204 100644
--- a/test/metabase/xrays/automagic_dashboards/core_test.clj
+++ b/test/metabase/xrays/automagic_dashboards/core_test.clj
@@ -1128,6 +1128,25 @@
(-> (#'magic/load-tables-with-enhanced-table-stats db-id nil)
first)))))))
+(deftest enhance-table-stats-honors-user-set-semantic-type-test
+ (testing "load-tables-with-enhanced-table-stats honors a user-set semantic_type that differs from the sync value"
+ (mt/with-temp [:model/Database {db-id :id} {}
+ :model/Table {table-id :id} {:db_id db-id}
+ :model/Field {pk-id :id} {:table_id table-id}
+ :model/Field _ {:table_id table-id}]
+ (mt/with-test-user :rasta
+ (automagic-dashboards.test/with-rollback-only-transaction
+ (testing "sanity check: neither Field is a PK by sync, so the Table isn't list-like"
+ (is (partial= {:list-like? false, :num-fields 2}
+ (-> (#'magic/load-tables-with-enhanced-table-stats db-id nil)
+ first))))
+ (mt/with-temp [:model/FieldUserSettings _ {:field_id pk-id
+ :semantic_type :type/PK
+ :semantic_type_set true}]
+ (is (partial= {:list-like? true, :num-fields 2}
+ (-> (#'magic/load-tables-with-enhanced-table-stats db-id nil)
+ first)))))))))
+
(deftest enhance-table-stats-fk-test
(mt/with-temp [:model/Database {db-id :id} {}
:model/Table {table-id :id} {:db_id db-id}
diff --git a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/orders/fields/created_at.yaml b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/orders/fields/created_at.yaml
index a211917ebb5..18450a39b27 100644
--- a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/orders/fields/created_at.yaml
+++ b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/orders/fields/created_at.yaml
@@ -12,7 +12,6 @@ effective_type: type/DateTimeWithLocalTZ
semantic_type: type/CreationTimestamp
dimensions: []
position: 7
-custom_position: 0
database_position: 7
serdes/meta:
- id: Sample Database
diff --git a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/orders/fields/discount.yaml b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/orders/fields/discount.yaml
index 112e0d246ae..560223d1979 100644
--- a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/orders/fields/discount.yaml
+++ b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/orders/fields/discount.yaml
@@ -12,7 +12,6 @@ effective_type: type/Float
semantic_type: type/Discount
dimensions: []
position: 6
-custom_position: 0
database_position: 6
serdes/meta:
- id: Sample Database
diff --git a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/orders/fields/id.yaml b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/orders/fields/id.yaml
index bcbcd0fde56..9808c0991be 100644
--- a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/orders/fields/id.yaml
+++ b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/orders/fields/id.yaml
@@ -13,7 +13,6 @@ semantic_type: type/PK
database_is_auto_increment: true
dimensions: []
position: 0
-custom_position: 0
database_position: 0
serdes/meta:
- id: Sample Database
diff --git a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/orders/fields/product_id.yaml b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/orders/fields/product_id.yaml
index 866d33c98a7..eba9b7181d7 100644
--- a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/orders/fields/product_id.yaml
+++ b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/orders/fields/product_id.yaml
@@ -17,7 +17,6 @@ fk_target_field_id:
- ID
dimensions: []
position: 2
-custom_position: 0
database_position: 2
serdes/meta:
- id: Sample Database
diff --git a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/orders/fields/quantity.yaml b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/orders/fields/quantity.yaml
index 79dc14f91be..7cd4ec12de8 100644
--- a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/orders/fields/quantity.yaml
+++ b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/orders/fields/quantity.yaml
@@ -12,7 +12,6 @@ effective_type: type/Integer
semantic_type: type/Quantity
dimensions: []
position: 8
-custom_position: 0
database_position: 8
has_field_values: auto-list
serdes/meta:
diff --git a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/orders/fields/subtotal.yaml b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/orders/fields/subtotal.yaml
index 81bce3623fe..f255d01d72c 100644
--- a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/orders/fields/subtotal.yaml
+++ b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/orders/fields/subtotal.yaml
@@ -11,7 +11,6 @@ base_type: type/Float
effective_type: type/Float
dimensions: []
position: 3
-custom_position: 0
database_position: 3
serdes/meta:
- id: Sample Database
diff --git a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/orders/fields/tax.yaml b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/orders/fields/tax.yaml
index 04a6b2c0c6f..bf61c128124 100644
--- a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/orders/fields/tax.yaml
+++ b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/orders/fields/tax.yaml
@@ -11,7 +11,6 @@ base_type: type/Float
effective_type: type/Float
dimensions: []
position: 4
-custom_position: 0
database_position: 4
serdes/meta:
- id: Sample Database
diff --git a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/orders/fields/total.yaml b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/orders/fields/total.yaml
index 674ba4a460f..0d7bece65db 100644
--- a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/orders/fields/total.yaml
+++ b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/orders/fields/total.yaml
@@ -11,7 +11,6 @@ base_type: type/Float
effective_type: type/Float
dimensions: []
position: 5
-custom_position: 0
database_position: 5
serdes/meta:
- id: Sample Database
diff --git a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/orders/fields/total___fieldusersettings.yaml b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/orders/fields/total___fieldusersettings.yaml
deleted file mode 100644
index 8eeab41c8cc..00000000000
--- a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/orders/fields/total___fieldusersettings.yaml
+++ /dev/null
@@ -1,13 +0,0 @@
-created_at: '2025-06-13T12:52:06.383265Z'
-description: Total order amount including tax
-serdes/meta:
-- id: Sample Database
- model: Database
-- id: PUBLIC
- model: Schema
-- id: ORDERS
- model: Table
-- id: TOTAL
- model: Field
-- id: '1'
- model: FieldUserSettings
diff --git a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/orders/fields/user_id.yaml b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/orders/fields/user_id.yaml
index 0f1728417eb..efe63e47a60 100644
--- a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/orders/fields/user_id.yaml
+++ b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/orders/fields/user_id.yaml
@@ -17,7 +17,6 @@ fk_target_field_id:
- ID
dimensions: []
position: 1
-custom_position: 0
database_position: 1
serdes/meta:
- id: Sample Database
diff --git a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/orders/orders___tableusersettings.yaml b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/orders/orders___tableusersettings.yaml
new file mode 100644
index 00000000000..f311ce86d54
--- /dev/null
+++ b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/orders/orders___tableusersettings.yaml
@@ -0,0 +1,26 @@
+created_at: '2025-06-13T12:52:06.383265Z'
+description: Every order placed, as the team describes it
+display_name: Customer Orders
+fields:
+- created_at: '2025-06-13T12:52:06.383265Z'
+ description: Total order amount including tax
+ serdes/meta:
+ - id: Sample Database
+ model: Database
+ - id: PUBLIC
+ model: Schema
+ - id: ORDERS
+ model: Table
+ - id: TOTAL
+ model: Field
+ - id: '1'
+ model: FieldUserSettings
+serdes/meta:
+- id: Sample Database
+ model: Database
+- id: PUBLIC
+ model: Schema
+- id: ORDERS
+ model: Table
+- id: '1'
+ model: TableUserSettings
diff --git a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/address.yaml b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/address.yaml
index 8799f1f85ca..38ef8a675c8 100644
--- a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/address.yaml
+++ b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/address.yaml
@@ -11,7 +11,6 @@ base_type: type/Text
effective_type: type/Text
dimensions: []
position: 1
-custom_position: 0
database_position: 1
serdes/meta:
- id: Sample Database
diff --git a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/birth_date.yaml b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/birth_date.yaml
index 4683d0d55c4..5cc06de0c26 100644
--- a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/birth_date.yaml
+++ b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/birth_date.yaml
@@ -11,7 +11,6 @@ base_type: type/Date
effective_type: type/Date
dimensions: []
position: 9
-custom_position: 0
database_position: 9
serdes/meta:
- id: Sample Database
diff --git a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/city.yaml b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/city.yaml
index 02a0085908a..c175c7edb19 100644
--- a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/city.yaml
+++ b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/city.yaml
@@ -12,7 +12,6 @@ effective_type: type/Text
semantic_type: type/City
dimensions: []
position: 5
-custom_position: 0
database_position: 5
serdes/meta:
- id: Sample Database
diff --git a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/created_at.yaml b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/created_at.yaml
index b052728ab8f..9e073b8d146 100644
--- a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/created_at.yaml
+++ b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/created_at.yaml
@@ -12,7 +12,6 @@ effective_type: type/DateTimeWithLocalTZ
semantic_type: type/CreationTimestamp
dimensions: []
position: 12
-custom_position: 0
database_position: 12
serdes/meta:
- id: Sample Database
diff --git a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/email.yaml b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/email.yaml
index 591a57ed486..b5dd900b912 100644
--- a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/email.yaml
+++ b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/email.yaml
@@ -12,7 +12,6 @@ effective_type: type/Text
semantic_type: type/Email
dimensions: []
position: 2
-custom_position: 0
database_position: 2
serdes/meta:
- id: Sample Database
diff --git a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/id.yaml b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/id.yaml
index 205d34845bc..099e5982c0a 100644
--- a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/id.yaml
+++ b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/id.yaml
@@ -13,7 +13,6 @@ semantic_type: type/PK
database_is_auto_increment: true
dimensions: []
position: 0
-custom_position: 0
database_position: 0
serdes/meta:
- id: Sample Database
diff --git a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/latitude.yaml b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/latitude.yaml
index 310961cf5ca..bc9ee452208 100644
--- a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/latitude.yaml
+++ b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/latitude.yaml
@@ -12,7 +12,6 @@ effective_type: type/Float
semantic_type: type/Latitude
dimensions: []
position: 11
-custom_position: 0
database_position: 11
serdes/meta:
- id: Sample Database
diff --git a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/longitude.yaml b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/longitude.yaml
index e76d154c4b5..0c0b5beb13c 100644
--- a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/longitude.yaml
+++ b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/longitude.yaml
@@ -12,7 +12,6 @@ effective_type: type/Float
semantic_type: type/Longitude
dimensions: []
position: 6
-custom_position: 0
database_position: 6
serdes/meta:
- id: Sample Database
diff --git a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/name.yaml b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/name.yaml
index 19670b2d6b5..1a8be929002 100644
--- a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/name.yaml
+++ b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/name.yaml
@@ -12,7 +12,6 @@ effective_type: type/Text
semantic_type: type/Name
dimensions: []
position: 4
-custom_position: 0
database_position: 4
serdes/meta:
- id: Sample Database
diff --git a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/password.yaml b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/password.yaml
index 9d9d4e53f8d..3dbb25b16f3 100644
--- a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/password.yaml
+++ b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/password.yaml
@@ -11,7 +11,6 @@ base_type: type/Text
effective_type: type/Text
dimensions: []
position: 3
-custom_position: 0
database_position: 3
serdes/meta:
- id: Sample Database
diff --git a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/source.yaml b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/source.yaml
index 1e9c4e7e222..1b3c4aba6f1 100644
--- a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/source.yaml
+++ b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/source.yaml
@@ -12,7 +12,6 @@ effective_type: type/Text
semantic_type: type/Source
dimensions: []
position: 8
-custom_position: 0
database_position: 8
has_field_values: auto-list
serdes/meta:
diff --git a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/state.yaml b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/state.yaml
index 1e0d6b9cd02..3f20a558a88 100644
--- a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/state.yaml
+++ b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/state.yaml
@@ -12,7 +12,6 @@ effective_type: type/Text
semantic_type: type/State
dimensions: []
position: 7
-custom_position: 0
database_position: 7
has_field_values: auto-list
serdes/meta:
diff --git a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/zip.yaml b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/zip.yaml
index b1a15f40504..63f3454dd58 100644
--- a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/zip.yaml
+++ b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/people/fields/zip.yaml
@@ -11,7 +11,6 @@ base_type: type/Text
effective_type: type/Text
dimensions: []
position: 10
-custom_position: 0
database_position: 10
serdes/meta:
- id: Sample Database
diff --git a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/fields/category.yaml b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/fields/category.yaml
index 18d7d3e4fbd..3d55f5e214e 100644
--- a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/fields/category.yaml
+++ b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/fields/category.yaml
@@ -12,7 +12,6 @@ effective_type: type/Text
semantic_type: type/Category
dimensions: []
position: 3
-custom_position: 0
database_position: 3
has_field_values: auto-list
serdes/meta:
diff --git a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/fields/created_at.yaml b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/fields/created_at.yaml
index 526c6bb421f..f236feb2f0c 100644
--- a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/fields/created_at.yaml
+++ b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/fields/created_at.yaml
@@ -12,7 +12,6 @@ effective_type: type/DateTimeWithLocalTZ
semantic_type: type/CreationTimestamp
dimensions: []
position: 7
-custom_position: 0
database_position: 7
serdes/meta:
- id: Sample Database
diff --git a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/fields/ean.yaml b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/fields/ean.yaml
index 3995a3c84ea..4a1d35d60ce 100644
--- a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/fields/ean.yaml
+++ b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/fields/ean.yaml
@@ -11,7 +11,6 @@ base_type: type/Text
effective_type: type/Text
dimensions: []
position: 1
-custom_position: 0
database_position: 1
has_field_values: auto-list
serdes/meta:
diff --git a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/fields/id.yaml b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/fields/id.yaml
index a9ba2b48d75..33a7f17ea9e 100644
--- a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/fields/id.yaml
+++ b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/fields/id.yaml
@@ -13,7 +13,6 @@ semantic_type: type/PK
database_is_auto_increment: true
dimensions: []
position: 0
-custom_position: 0
database_position: 0
serdes/meta:
- id: Sample Database
diff --git a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/fields/price.yaml b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/fields/price.yaml
index c9ce63d32fa..69c3e8ccbc4 100644
--- a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/fields/price.yaml
+++ b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/fields/price.yaml
@@ -11,7 +11,6 @@ base_type: type/Float
effective_type: type/Float
dimensions: []
position: 5
-custom_position: 0
database_position: 5
serdes/meta:
- id: Sample Database
diff --git a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/fields/rating.yaml b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/fields/rating.yaml
index 2ff80cc8807..fab797225bd 100644
--- a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/fields/rating.yaml
+++ b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/fields/rating.yaml
@@ -12,7 +12,6 @@ effective_type: type/Float
semantic_type: type/Score
dimensions: []
position: 6
-custom_position: 0
database_position: 6
serdes/meta:
- id: Sample Database
diff --git a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/fields/title.yaml b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/fields/title.yaml
index d144cc26791..efd44ecacc0 100644
--- a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/fields/title.yaml
+++ b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/fields/title.yaml
@@ -12,7 +12,6 @@ effective_type: type/Text
semantic_type: type/Title
dimensions: []
position: 2
-custom_position: 0
database_position: 2
has_field_values: auto-list
serdes/meta:
diff --git a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/fields/vendor.yaml b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/fields/vendor.yaml
index 581d6982367..be517e9c583 100644
--- a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/fields/vendor.yaml
+++ b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/fields/vendor.yaml
@@ -12,7 +12,6 @@ effective_type: type/Text
semantic_type: type/Company
dimensions: []
position: 4
-custom_position: 0
database_position: 4
has_field_values: auto-list
serdes/meta:
diff --git a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/reviews/fields/body.yaml b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/reviews/fields/body.yaml
index b034f9dea89..70c5694b3cc 100644
--- a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/reviews/fields/body.yaml
+++ b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/reviews/fields/body.yaml
@@ -12,7 +12,6 @@ effective_type: type/Text
dimensions: []
preview_display: false
position: 4
-custom_position: 0
database_position: 4
serdes/meta:
- id: Sample Database
diff --git a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/reviews/fields/created_at.yaml b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/reviews/fields/created_at.yaml
index 7e40538f59e..eedba1e085f 100644
--- a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/reviews/fields/created_at.yaml
+++ b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/reviews/fields/created_at.yaml
@@ -12,7 +12,6 @@ effective_type: type/DateTimeWithLocalTZ
semantic_type: type/CreationTimestamp
dimensions: []
position: 5
-custom_position: 0
database_position: 5
serdes/meta:
- id: Sample Database
diff --git a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/reviews/fields/id.yaml b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/reviews/fields/id.yaml
index da92b75c923..c43d444ec3c 100644
--- a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/reviews/fields/id.yaml
+++ b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/reviews/fields/id.yaml
@@ -13,7 +13,6 @@ semantic_type: type/PK
database_is_auto_increment: true
dimensions: []
position: 0
-custom_position: 0
database_position: 0
serdes/meta:
- id: Sample Database
diff --git a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/reviews/fields/product_id.yaml b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/reviews/fields/product_id.yaml
index fa1cb1448b4..18852dde9ca 100644
--- a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/reviews/fields/product_id.yaml
+++ b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/reviews/fields/product_id.yaml
@@ -17,7 +17,6 @@ fk_target_field_id:
- ID
dimensions: []
position: 1
-custom_position: 0
database_position: 1
serdes/meta:
- id: Sample Database
diff --git a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/reviews/fields/rating.yaml b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/reviews/fields/rating.yaml
index 1922f31c087..60b41013847 100644
--- a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/reviews/fields/rating.yaml
+++ b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/reviews/fields/rating.yaml
@@ -12,7 +12,6 @@ effective_type: type/Integer
semantic_type: type/Score
dimensions: []
position: 3
-custom_position: 0
database_position: 3
has_field_values: auto-list
serdes/meta:
diff --git a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/reviews/fields/reviewer.yaml b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/reviews/fields/reviewer.yaml
index d608a4526ee..b57431f7626 100644
--- a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/reviews/fields/reviewer.yaml
+++ b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/reviews/fields/reviewer.yaml
@@ -11,7 +11,6 @@ base_type: type/Text
effective_type: type/Text
dimensions: []
position: 2
-custom_position: 0
database_position: 2
serdes/meta:
- id: Sample Database
diff --git a/test_resources/yaml_checks/databases/SQLite DB/tables/orders/fields/id.yaml b/test_resources/yaml_checks/databases/SQLite DB/tables/orders/fields/id.yaml
index 34fcc5de514..ce712c02f93 100644
--- a/test_resources/yaml_checks/databases/SQLite DB/tables/orders/fields/id.yaml
+++ b/test_resources/yaml_checks/databases/SQLite DB/tables/orders/fields/id.yaml
@@ -21,7 +21,6 @@ parent_id: null
coercion_strategy: null
preview_display: true
position: 0
-custom_position: 0
database_position: 0
has_field_values: null
settings: null
diff --git a/test_resources/yaml_checks/databases/SQLite DB/tables/orders/fields/user_id.yaml b/test_resources/yaml_checks/databases/SQLite DB/tables/orders/fields/user_id.yaml
index 53ab777bb7e..e538bd53d30 100644
--- a/test_resources/yaml_checks/databases/SQLite DB/tables/orders/fields/user_id.yaml
+++ b/test_resources/yaml_checks/databases/SQLite DB/tables/orders/fields/user_id.yaml
@@ -25,7 +25,6 @@ parent_id: null
coercion_strategy: null
preview_display: true
position: 1
-custom_position: 0
database_position: 1
has_field_values: null
settings: null
diff --git a/test_resources/yaml_checks/databases/SQLite DB/tables/users/fields/id.yaml b/test_resources/yaml_checks/databases/SQLite DB/tables/users/fields/id.yaml
index 6a2e07a3e2d..f76207ea4bb 100644
--- a/test_resources/yaml_checks/databases/SQLite DB/tables/users/fields/id.yaml
+++ b/test_resources/yaml_checks/databases/SQLite DB/tables/users/fields/id.yaml
@@ -21,7 +21,6 @@ parent_id: null
coercion_strategy: null
preview_display: true
position: 0
-custom_position: 0
database_position: 0
has_field_values: null
settings: null
diff --git a/test_resources/yaml_checks/databases/SQLite DB/tables/users/fields/name.yaml b/test_resources/yaml_checks/databases/SQLite DB/tables/users/fields/name.yaml
index a48af8a14f8..79951f8739d 100644
--- a/test_resources/yaml_checks/databases/SQLite DB/tables/users/fields/name.yaml
+++ b/test_resources/yaml_checks/databases/SQLite DB/tables/users/fields/name.yaml
@@ -21,7 +21,6 @@ parent_id: null
coercion_strategy: null
preview_display: true
position: 1
-custom_position: 0
database_position: 1
has_field_values: auto-list
settings: null
diff --git a/test_resources/yaml_checks/databases/Test Database/schemas/public/tables/orders/fields/created_at.yaml b/test_resources/yaml_checks/databases/Test Database/schemas/public/tables/orders/fields/created_at.yaml
index 29f6ed79e2e..cbdbf30596f 100644
--- a/test_resources/yaml_checks/databases/Test Database/schemas/public/tables/orders/fields/created_at.yaml
+++ b/test_resources/yaml_checks/databases/Test Database/schemas/public/tables/orders/fields/created_at.yaml
@@ -21,7 +21,6 @@ parent_id: null
coercion_strategy: null
preview_display: true
position: 3
-custom_position: 0
database_position: 3
has_field_values: null
settings: null
diff --git a/test_resources/yaml_checks/databases/Test Database/schemas/public/tables/orders/fields/id.yaml b/test_resources/yaml_checks/databases/Test Database/schemas/public/tables/orders/fields/id.yaml
index 523095c67a9..efd8495fcfc 100644
--- a/test_resources/yaml_checks/databases/Test Database/schemas/public/tables/orders/fields/id.yaml
+++ b/test_resources/yaml_checks/databases/Test Database/schemas/public/tables/orders/fields/id.yaml
@@ -21,7 +21,6 @@ parent_id: null
coercion_strategy: null
preview_display: true
position: 0
-custom_position: 0
database_position: 0
has_field_values: null
settings: null
diff --git a/test_resources/yaml_checks/databases/Test Database/schemas/public/tables/orders/fields/product_id.yaml b/test_resources/yaml_checks/databases/Test Database/schemas/public/tables/orders/fields/product_id.yaml
index dad77cd1f6b..cb903389f3c 100644
--- a/test_resources/yaml_checks/databases/Test Database/schemas/public/tables/orders/fields/product_id.yaml
+++ b/test_resources/yaml_checks/databases/Test Database/schemas/public/tables/orders/fields/product_id.yaml
@@ -25,7 +25,6 @@ parent_id: null
coercion_strategy: null
preview_display: true
position: 2
-custom_position: 0
database_position: 2
has_field_values: null
settings: null
diff --git a/test_resources/yaml_checks/databases/Test Database/schemas/public/tables/orders/fields/total.yaml b/test_resources/yaml_checks/databases/Test Database/schemas/public/tables/orders/fields/total.yaml
index 7dc2eb020e0..061d035c4b9 100644
--- a/test_resources/yaml_checks/databases/Test Database/schemas/public/tables/orders/fields/total.yaml
+++ b/test_resources/yaml_checks/databases/Test Database/schemas/public/tables/orders/fields/total.yaml
@@ -21,7 +21,6 @@ parent_id: null
coercion_strategy: null
preview_display: true
position: 1
-custom_position: 0
database_position: 1
has_field_values: null
settings: null
diff --git a/test_resources/yaml_checks/databases/Test Database/schemas/public/tables/products/fields/id.yaml b/test_resources/yaml_checks/databases/Test Database/schemas/public/tables/products/fields/id.yaml
index cd6fb4cb53c..04fe3bd2ed2 100644
--- a/test_resources/yaml_checks/databases/Test Database/schemas/public/tables/products/fields/id.yaml
+++ b/test_resources/yaml_checks/databases/Test Database/schemas/public/tables/products/fields/id.yaml
@@ -21,7 +21,6 @@ parent_id: null
coercion_strategy: null
preview_display: true
position: 0
-custom_position: 0
database_position: 0
has_field_values: null
settings: null
diff --git a/test_resources/yaml_checks/databases/Test Database/schemas/public/tables/products/fields/price.yaml b/test_resources/yaml_checks/databases/Test Database/schemas/public/tables/products/fields/price.yaml
index 18c888b5383..76509ca9170 100644
--- a/test_resources/yaml_checks/databases/Test Database/schemas/public/tables/products/fields/price.yaml
+++ b/test_resources/yaml_checks/databases/Test Database/schemas/public/tables/products/fields/price.yaml
@@ -21,7 +21,6 @@ parent_id: null
coercion_strategy: null
preview_display: true
position: 2
-custom_position: 0
database_position: 2
has_field_values: null
settings: null
diff --git a/test_resources/yaml_checks/databases/Test Database/schemas/public/tables/products/fields/title.yaml b/test_resources/yaml_checks/databases/Test Database/schemas/public/tables/products/fields/title.yaml
index 87e204b4c19..c3d71e7f9ca 100644
--- a/test_resources/yaml_checks/databases/Test Database/schemas/public/tables/products/fields/title.yaml
+++ b/test_resources/yaml_checks/databases/Test Database/schemas/public/tables/products/fields/title.yaml
@@ -21,7 +21,6 @@ parent_id: null
coercion_strategy: null
preview_display: true
position: 1
-custom_position: 0
database_position: 1
has_field_values: null
settings: null