The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Only unfold JSON columns the sync user can read (#84049)
dpsuttonf707cf653783
diff --git a/src/metabase/driver/sql_jdbc/sync/describe_table.clj b/src/metabase/driver/sql_jdbc/sync/describe_table.cljindex 868a156f284..01563c990c3 100644--- a/src/metabase/driver/sql_jdbc/sync/describe_table.clj+++ b/src/metabase/driver/sql_jdbc/sync/describe_table.clj@@ -746,9 +746,29 @@:nfc-path field-path}))(into #{})))+(defn- table-fields-for-json-unfolding+ "Columns of `table` to consider for JSON unfolding. For drivers with `:describe-fields`, runs [[describe-fields-sql]],+ which can list only the columns the connection user is allowed to read (e.g. Postgres reads+ `information_schema.columns`, which hides columns without a column-level grant), whereas JDBC `getColumns` lists them+ all. Sampling an unreadable column fails the whole query (#83790)."+ [driver ^Connection conn database table]+ (if (driver/database-supports? driver :describe-fields database)+ ;; on `conn` rather than through `driver/describe-fields`, which checks out more pooled connections and runs+ ;; pre-processing JSON detection doesn't need (e.g. Postgres's enum lookup)+ (let [sql-args (m/mapply describe-fields-sql driver (cond-> {:table-names [(:name table)]+ :details (driver.conn/effective-details database)}+ (:schema table) (assoc :schema-names [(:schema table)])))]+ (into []+ (comp+ ;; `:table-names` can match more than this table, e.g. MySQL compares names without case+ (filter #(= (:name table) (:table-name %)))+ (describe-fields-xf driver database))+ (jdbc/reducible-query {:connection conn} sql-args {:identifiers identity})))+ (describe-table-fields driver conn table nil)))+(defn- table->unfold-json-fields"Given a table return a list of json fields that need to unfold."- [driver conn table]+ [driver conn database table](let [fields-with-json-unfolding-disabled(->> (driver.db/json-field-names-with-unfolding-disabled (u/the-id table));; in a delay so we'll query only if there's at least one json field@@ -758,7 +778,7 @@(filter #(isa? (:base-type %) :type/JSON))(remove #(contains? @fields-with-json-unfolding-disabled (:name %)))(describe-table-fields-xf driver table))- (describe-table-fields driver conn table nil))))+ (table-fields-for-json-unfolding driver conn database table))))(defn- sample-json-row-honey-sql"Return a honeysql query used to get row sample to describe json columns.@@ -837,7 +857,7 @@jdbc-specnil(fn [^Connection conn]- (let [unfold-json-fields (table->unfold-json-fields driver conn table)+ (let [unfold-json-fields (table->unfold-json-fields driver conn database table);; Just pass in `nil` here, that's what we do in the normal sync process and it seems to work correctly.;; We don't currently have a driver-agnostic way to get the physical database name. `(:name database)` is;; wrong, because it's a human-friendly name rather than a physical name. `(get-indiff --git a/test/metabase/driver/mysql_test.clj b/test/metabase/driver/mysql_test.cljindex c69e45b14ac..9d7a99e8e12 100644--- a/test/metabase/driver/mysql_test.clj+++ b/test/metabase/driver/mysql_test.clj@@ -787,6 +787,24 @@database(t2/select-one :model/Table :db_id (mt/id) :name "json_table"))))))))))))+(deftest json-unfolding-ignores-table-differing-only-in-case-test+ (testing "JSON unfolding lists only the synced table's columns, not a table's differing only in case (#83790)"+ (mt/test-driver :mysql+ (when-not (mysql/mariadb? (mt/db))+ (tx/drop-if-exists-and-create-db! driver/*driver* "json_case_test")+ (let [details (mt/dbdef->connection-details driver/*driver* :db {:database-name "json_case_test"})+ spec (sql-jdbc.conn/connection-details->spec driver/*driver* details)]+ (doseq [statement ["CREATE TABLE `orders` (`id` INT PRIMARY KEY, `data` JSON);"+ "CREATE TABLE `Orders` (`id` INT PRIMARY KEY, `payload` JSON);"+ "INSERT INTO `orders` VALUES (1, '{\"a\": 1}');"]]+ (jdbc/execute! spec [statement]))+ (mt/with-temp [:model/Database database {:engine driver/*driver* :details details}]+ (sync/sync-database! database {:scan :schema})+ (is (= #{"id" "data" "data → a"}+ (t2/select-fn-set :name :model/Field+ :table_id (t2/select-one-pk :model/Table :db_id (:id database) :name "orders")+ :active true)))))))))+(deftest json-alias-test(mt/test-driver :mysql(when (not (mysql/mariadb? (mt/db)))diff --git a/test/metabase/driver/postgres_test.clj b/test/metabase/driver/postgres_test.cljindex ac3eb8c2798..4d0fa86930a 100644--- a/test/metabase/driver/postgres_test.clj+++ b/test/metabase/driver/postgres_test.clj@@ -1662,6 +1662,43 @@(is (= #{"table_with_perms"}(t2/select-fn-set :name :model/Table :db_id (:id database)))))))))+(deftest json-unfolding-skips-columns-without-select-privilege-test+ (testing "JSON unfolding only samples JSON columns the connection user can read, so column-level grants don't break field sync (#83790)"+ (mt/test-driver :postgres+ (tx/drop-if-exists-and-create-db! driver/*driver* "json-column-grants-test")+ (let [role (str "mb_restricted_" (u/lower-case-en (mt/random-name)))+ details (mt/dbdef->connection-details :postgres :db {:database-name "json-column-grants-test"+ :json-unfolding true})+ spec (sql-jdbc.conn/connection-details->spec :postgres details)+ active-fields (fn [database]+ (t2/select-fn-set :name :model/Field+ :table_id (t2/select-one-pk :model/Table :db_id (:id database) :name "table_a")+ :active true))]+ (doseq [statement ["CREATE TABLE public.table_a (id INTEGER PRIMARY KEY, col_visible TEXT, col_json JSONB, col_json_hidden JSONB, col_hidden TEXT);"+ "INSERT INTO public.table_a VALUES (1, 'a', '{\"key_a\": 1}', '{\"key_b\": 2}', 'b');"+ (format "CREATE ROLE %s LOGIN PASSWORD 'password';" role)+ (format "GRANT SELECT (id, col_visible, col_json) ON public.table_a TO %s;" role)]]+ (jdbc/execute! spec [statement]))+ (try+ (mt/with-temp [:model/Database database {:engine :postgres, :details details}]+ (testing "sanity check: a user that can read every column syncs all of them"+ (sync/sync-database! database {:scan :schema})+ (is (= #{"id" "col_visible" "col_json" "col_json → key_a" "col_json_hidden" "col_json_hidden → key_b" "col_hidden"}+ (active-fields database))))+ (t2/update! :model/Database (:id database) {:details (assoc details :user role :password "password")})+ (let [database (t2/select-one :model/Database (:id database))]+ (binding [sync-util/*log-exceptions-and-continue?* false]+ (sync/sync-database! database {:scan :schema}))+ (testing "after switching to a user with column-level grants, unreadable columns are retired and readable JSON is still unfolded"+ (is (= #{"id" "col_visible" "col_json" "col_json → key_a"}+ (active-fields database))))+ (testing "a raw query on the table runs"+ (mt/with-db database+ (is (= 1 (count (mt/rows (mt/run-mbql-query table_a)))))))))+ (finally+ (jdbc/execute! spec [(format "DROP OWNED BY %s;" role)])+ (jdbc/execute! spec [(format "DROP ROLE %s;" role)])))))))+(deftest json-operator-?-works(testing "Make sure the Postgres ? operators (for JSON types) work in native queries"(mt/test-driver :postgres