Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Only unfold JSON columns the sync user can read (#84049)

dpsuttonf707cf653783
diff --git a/src/metabase/driver/sql_jdbc/sync/describe_table.clj b/src/metabase/driver/sql_jdbc/sync/describe_table.clj
index 868a156f284..01563c990c3 100644
--- a/src/metabase/driver/sql_jdbc/sync/describe_table.clj
+++ b/src/metabase/driver/sql_jdbc/sync/describe_table.clj
@@ -746,9 +746,29 @@
:nfc-path field-path}))
(into #{})))
+(defn- table-fields-for-json-unfolding
+ "Columns of `table` to consider for JSON unfolding. For drivers with `:describe-fields`, runs [[describe-fields-sql]],
+ which can list only the columns the connection user is allowed to read (e.g. Postgres reads
+ `information_schema.columns`, which hides columns without a column-level grant), whereas JDBC `getColumns` lists them
+ all. Sampling an unreadable column fails the whole query (#83790)."
+ [driver ^Connection conn database table]
+ (if (driver/database-supports? driver :describe-fields database)
+ ;; on `conn` rather than through `driver/describe-fields`, which checks out more pooled connections and runs
+ ;; pre-processing JSON detection doesn't need (e.g. Postgres's enum lookup)
+ (let [sql-args (m/mapply describe-fields-sql driver (cond-> {:table-names [(:name table)]
+ :details (driver.conn/effective-details database)}
+ (:schema table) (assoc :schema-names [(:schema table)])))]
+ (into []
+ (comp
+ ;; `:table-names` can match more than this table, e.g. MySQL compares names without case
+ (filter #(= (:name table) (:table-name %)))
+ (describe-fields-xf driver database))
+ (jdbc/reducible-query {:connection conn} sql-args {:identifiers identity})))
+ (describe-table-fields driver conn table nil)))
+
(defn- table->unfold-json-fields
"Given a table return a list of json fields that need to unfold."
- [driver conn table]
+ [driver conn database table]
(let [fields-with-json-unfolding-disabled
(->> (driver.db/json-field-names-with-unfolding-disabled (u/the-id table))
;; in a delay so we'll query only if there's at least one json field
@@ -758,7 +778,7 @@
(filter #(isa? (:base-type %) :type/JSON))
(remove #(contains? @fields-with-json-unfolding-disabled (:name %)))
(describe-table-fields-xf driver table))
- (describe-table-fields driver conn table nil))))
+ (table-fields-for-json-unfolding driver conn database table))))
(defn- sample-json-row-honey-sql
"Return a honeysql query used to get row sample to describe json columns.
@@ -837,7 +857,7 @@
jdbc-spec
nil
(fn [^Connection conn]
- (let [unfold-json-fields (table->unfold-json-fields driver conn table)
+ (let [unfold-json-fields (table->unfold-json-fields driver conn database table)
;; Just pass in `nil` here, that's what we do in the normal sync process and it seems to work correctly.
;; We don't currently have a driver-agnostic way to get the physical database name. `(:name database)` is
;; wrong, because it's a human-friendly name rather than a physical name. `(get-in
diff --git a/test/metabase/driver/mysql_test.clj b/test/metabase/driver/mysql_test.clj
index c69e45b14ac..9d7a99e8e12 100644
--- a/test/metabase/driver/mysql_test.clj
+++ b/test/metabase/driver/mysql_test.clj
@@ -787,6 +787,24 @@
database
(t2/select-one :model/Table :db_id (mt/id) :name "json_table"))))))))))))
+(deftest json-unfolding-ignores-table-differing-only-in-case-test
+ (testing "JSON unfolding lists only the synced table's columns, not a table's differing only in case (#83790)"
+ (mt/test-driver :mysql
+ (when-not (mysql/mariadb? (mt/db))
+ (tx/drop-if-exists-and-create-db! driver/*driver* "json_case_test")
+ (let [details (mt/dbdef->connection-details driver/*driver* :db {:database-name "json_case_test"})
+ spec (sql-jdbc.conn/connection-details->spec driver/*driver* details)]
+ (doseq [statement ["CREATE TABLE `orders` (`id` INT PRIMARY KEY, `data` JSON);"
+ "CREATE TABLE `Orders` (`id` INT PRIMARY KEY, `payload` JSON);"
+ "INSERT INTO `orders` VALUES (1, '{\"a\": 1}');"]]
+ (jdbc/execute! spec [statement]))
+ (mt/with-temp [:model/Database database {:engine driver/*driver* :details details}]
+ (sync/sync-database! database {:scan :schema})
+ (is (= #{"id" "data" "data → a"}
+ (t2/select-fn-set :name :model/Field
+ :table_id (t2/select-one-pk :model/Table :db_id (:id database) :name "orders")
+ :active true)))))))))
+
(deftest json-alias-test
(mt/test-driver :mysql
(when (not (mysql/mariadb? (mt/db)))
diff --git a/test/metabase/driver/postgres_test.clj b/test/metabase/driver/postgres_test.clj
index ac3eb8c2798..4d0fa86930a 100644
--- a/test/metabase/driver/postgres_test.clj
+++ b/test/metabase/driver/postgres_test.clj
@@ -1662,6 +1662,43 @@
(is (= #{"table_with_perms"}
(t2/select-fn-set :name :model/Table :db_id (:id database)))))))))
+(deftest json-unfolding-skips-columns-without-select-privilege-test
+ (testing "JSON unfolding only samples JSON columns the connection user can read, so column-level grants don't break field sync (#83790)"
+ (mt/test-driver :postgres
+ (tx/drop-if-exists-and-create-db! driver/*driver* "json-column-grants-test")
+ (let [role (str "mb_restricted_" (u/lower-case-en (mt/random-name)))
+ details (mt/dbdef->connection-details :postgres :db {:database-name "json-column-grants-test"
+ :json-unfolding true})
+ spec (sql-jdbc.conn/connection-details->spec :postgres details)
+ active-fields (fn [database]
+ (t2/select-fn-set :name :model/Field
+ :table_id (t2/select-one-pk :model/Table :db_id (:id database) :name "table_a")
+ :active true))]
+ (doseq [statement ["CREATE TABLE public.table_a (id INTEGER PRIMARY KEY, col_visible TEXT, col_json JSONB, col_json_hidden JSONB, col_hidden TEXT);"
+ "INSERT INTO public.table_a VALUES (1, 'a', '{\"key_a\": 1}', '{\"key_b\": 2}', 'b');"
+ (format "CREATE ROLE %s LOGIN PASSWORD 'password';" role)
+ (format "GRANT SELECT (id, col_visible, col_json) ON public.table_a TO %s;" role)]]
+ (jdbc/execute! spec [statement]))
+ (try
+ (mt/with-temp [:model/Database database {:engine :postgres, :details details}]
+ (testing "sanity check: a user that can read every column syncs all of them"
+ (sync/sync-database! database {:scan :schema})
+ (is (= #{"id" "col_visible" "col_json" "col_json → key_a" "col_json_hidden" "col_json_hidden → key_b" "col_hidden"}
+ (active-fields database))))
+ (t2/update! :model/Database (:id database) {:details (assoc details :user role :password "password")})
+ (let [database (t2/select-one :model/Database (:id database))]
+ (binding [sync-util/*log-exceptions-and-continue?* false]
+ (sync/sync-database! database {:scan :schema}))
+ (testing "after switching to a user with column-level grants, unreadable columns are retired and readable JSON is still unfolded"
+ (is (= #{"id" "col_visible" "col_json" "col_json → key_a"}
+ (active-fields database))))
+ (testing "a raw query on the table runs"
+ (mt/with-db database
+ (is (= 1 (count (mt/rows (mt/run-mbql-query table_a)))))))))
+ (finally
+ (jdbc/execute! spec [(format "DROP OWNED BY %s;" role)])
+ (jdbc/execute! spec [(format "DROP ROLE %s;" role)])))))))
+
(deftest json-operator-?-works
(testing "Make sure the Postgres ? operators (for JSON types) work in native queries"
(mt/test-driver :postgres