The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Revert the revert of column validation for incremental transforms (#68460)
Dan Stonea2c4eef9dece
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/api.clj b/enterprise/backend/src/metabase_enterprise/transforms/api.cljindex 3e6c465a616..cd71361aa82 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/api.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/api.clj@@ -20,6 +20,7 @@[metabase.driver.sql-jdbc.execute :as sql-jdbc.execute][metabase.driver.util :as driver.u][metabase.events.core :as events]+ [metabase.lib.core :as lib][metabase.query-processor.compile :as qp.compile][metabase.query-processor.schema :as qp.schema][metabase.request.core :as request]@@ -122,6 +123,83 @@[:tag_ids {:optional true} [:maybe (ms/QueryVectorOf ms/IntGreaterThanOrEqualToZero)]]]](get-transforms query-params))+(defn- extract-all-columns-from-query+ "Extracts column metadata (name and type) from a query.++ Returns a sequence of maps with `:name` and `:base_type` keys, or nil if extraction fails.++ The query is first compiled to native SQL, then uses PreparedStatement.getMetaData()+ to inspect the query structure. This works for most modern JDBC drivers but may not+ be supported by all drivers or for all query types."+ [driver database-id query]+ (try+ (let [{:keys [query]} (qp.compile/compile query)]+ (sql-jdbc.execute/do-with-connection-with-options+ driver+ database-id+ {}+ (fn [conn]+ (with-open [^PreparedStatement stmt (sql-jdbc.execute/prepared-statement driver conn query [])]+ (when-let [rsmeta (.getMetaData stmt)]+ (seq (sql-jdbc.execute/column-metadata driver rsmeta)))))))+ (catch Exception e+ (log/debugf e "Failed to extract columns from query: %s" (ex-message e))+ nil)))++(defn- extract-incremental-filter-columns-from-query+ "Extracts column names suitable for incremental transform checkpoint filtering.++ This function is specifically for incremental transform checkpoint column selection.+ It only returns columns with types supported for checkpoint filtering:+ - Temporal types (timestamp, timestamp with timezone)+ - Numeric types (integer, float, decimal)++ Text, boolean, and other types are filtered out as they are not supported for+ incremental checkpointing.++ Returns a vector of column names (as strings), or nil if extraction fails.++ The query is first compiled to native SQL, then uses PreparedStatement.getMetaData()+ to inspect the query structure. This works for most modern JDBC drivers but may not+ be supported by all drivers or for all query types."+ [driver database-id query]+ (some->> (extract-all-columns-from-query driver database-id query)+ (filter (comp transforms.util/supported-incremental-filter-type? :base_type))+ (mapv :name)))++(defn- validate-incremental-column-type!+ "Validates that the checkpoint column for an incremental transform has a supported type.++ For MBQL/Python transforms, resolves the column from the query using the unique key.+ For native queries, extracts columns from the query and checks the checkpoint-filter column.++ Throws a 400 error if the column type is not supported or cannot be resolved."+ [{:keys [source]}]+ (when-let [{:keys [checkpoint-filter checkpoint-filter-unique-key] strategy-type :type}+ (:source-incremental-strategy source)]+ (when (and (= :query (:type source)) (= "checkpoint" strategy-type))+ (let [{:keys [query]} source+ database-id (:database query)+ database (api/check-404 (t2/select-one :model/Database :id database-id))+ driver-name (driver/the-initialized-driver (:engine database))]+ (cond+ ;; For MBQL, resolve column from query metadata+ checkpoint-filter-unique-key+ (let [column (lib/column-with-unique-key query checkpoint-filter-unique-key)]+ (api/check-400 column (deferred-tru "Checkpoint column not found in query."))+ (api/check-400 (transforms.util/supported-incremental-filter-type? (:base-type column))+ (deferred-tru "Checkpoint column type {0} is not supported. Only numeric and temporal types are supported for incremental filtering."+ (pr-str (:base-type column)))))++ ;; For native query with checkpoint-filter, validate type if we can extract the column metadata+ checkpoint-filter+ (when-some [column-metadata (seq (extract-all-columns-from-query driver-name database-id query))]+ (when-some [column (first (filter #(= checkpoint-filter (:name %)) column-metadata))]+ (api/check-400 (transforms.util/supported-incremental-filter-type? (:base_type column))+ (deferred-tru "Checkpoint column ''{0}'' has unsupported type {1}. Only numeric and temporal columns are supported for incremental filtering."+ checkpoint-filter+ (pr-str (:base_type column)))))))))))+;; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to;; use our API + we will need it when we make auto-TypeScript-signature generation happen;;@@ -143,6 +221,7 @@(api/create-check :model/Transform body)(check-database-feature body)(check-feature-enabled! body)+ (validate-incremental-column-type! body)(api/check (not (transforms.util/target-table-exists? body))403@@ -256,6 +335,7 @@;; we must validate on a full transform object(check-feature-enabled! new)(check-database-feature new)+ (validate-incremental-column-type! new)(when (transforms.util/query-transform? old)(when-let [{:keys [cycle-str]} (transforms.ordering/get-transform-cycle new)](throw (ex-info (str "Cyclic transform definitions detected: " cycle-str)@@ -338,30 +418,6 @@:run_id run-id})(assoc :status 202)))))-(defn- extract-columns-from-query- "Attempts to extract column names from an MBQL query without executing it.-- Returns a vector of column names (as strings), or nil if extraction fails.-- The query is first compiled to native SQL, hen uses PreparedStatement.getMetaData()- to inspect the query structure. This works for most modern JDBC drivers but may not- be supported by all drivers or for all query types."- [driver database-id query]- (try- (let [{:keys [query]} (qp.compile/compile query)]- (sql-jdbc.execute/do-with-connection-with-options- driver- database-id- {}- (fn [conn]- (with-open [^PreparedStatement stmt (sql-jdbc.execute/prepared-statement driver conn query [])]- (when-let [rsmeta (.getMetaData stmt)]- (let [columns (sql-jdbc.execute/column-metadata driver rsmeta)]- (seq (mapv :name columns))))))))- (catch Exception e- (log/debugf e "Failed to extract columns from query: %s" (ex-message e))- nil)))-(defn- simple-native-query?"Checks if a native SQL query string is simple enough for automatic checkpoint insertion."[sql-string]@@ -402,16 +458,19 @@(api/check-superuser)(simple-native-query? query))-;; TODO (Cam 2025-12-04) please add a response schema to this API endpoint, it makes it easier for our customers to-;; use our API + we will need it when we make auto-TypeScript-signature generation happen-;;-#_{:clj-kondo/ignore [:metabase/validate-defendpoint-has-response-schema]}(api.macros/defendpoint :post "/extract-columns"- "Extract column names from an MBQL query without executing it.+ :- [:map [:columns [:maybe [:sequential :string]]]]+ "Extract column names suitable for incremental transform checkpoint filtering.++ This endpoint is specifically for populating the checkpoint column dropdown in+ incremental transforms. It only returns columns with types supported for checkpoint+ filtering: temporal (timestamp/tz) and numeric (int/float) types.++ Text, boolean, and other unsupported column types are filtered out.- Compiles the query to native SQL using [[qp.compile/compile-with-inline-parameters]],+ The query is compiled to native SQL using [[qp.compile/compile-with-inline-parameters]],which handles parameterized queries with template tags. Then extracts column names- using PreparedStatement metadata.+ and types using PreparedStatement metadata.Returns a map with a :columns key containing a vector of column names (strings).If extraction fails, returns nil for :columns."@@ -421,9 +480,9 @@[:query ::qp.schema/any-query]]](api/check-superuser)(let [database-id (:database query)- database (api/check-404 (t2/select-one :model/Database :id database-id))+ database (api/check-404 (t2/select-one :model/Database :id database-id))driver-name (driver/the-initialized-driver (:engine database))- columns (extract-columns-from-query driver-name database-id query)]+ columns (extract-incremental-filter-columns-from-query driver-name database-id query)]{:columns columns}))(def ^{:arglists '([request respond raise])} routesdiff --git a/enterprise/backend/src/metabase_enterprise/transforms/util.clj b/enterprise/backend/src/metabase_enterprise/transforms/util.cljindex 375cf0e8f98..068fdf7cd0a 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/util.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/util.clj@@ -214,6 +214,14 @@[source](= :checkpoint (some-> source :source-incremental-strategy :type keyword)))+(defn supported-incremental-filter-type?+ "Returns true if the given base-type is supported for incremental filtering.++ We only support temporal (timestamp/tz) and numeric (int/float) types."+ [base-type]+ (or (isa? base-type :type/Temporal)+ (isa? base-type :type/Number)))+(defn- source->checkpoint-filter-unique-key"Extract the checkpoint filter column from `query` using the unique key specified in `source-incremental-strategy`."[query source-incremental-strategy]@@ -223,15 +231,29 @@"Resolve the checkpoint filter column for an incremental transform.Tries to resolve the column using the unique key first.- Falls back to looking up the column by name from the target table if a `:checkpoint-filter` is specified."+ Falls back to looking up the column by name from the target table if a `:checkpoint-filter` is specified.++ Validates that the resolved column has a supported type for checkpoint filtering (numeric or temporal).+ Throws an exception if the column type is not supported."[query source-incremental-strategy table metadata-provider]- (or- (source->checkpoint-filter-unique-key query source-incremental-strategy)- (when-let [field-name (-> source-incremental-strategy :checkpoint-filter)]- (when-let [field-id (t2/select-one-pk :model/Field- :table_id (:id table)- :name field-name)]- (lib.metadata/field metadata-provider field-id)))))+ (let [{:keys [checkpoint-filter checkpoint-filter-unique-key]} source-incremental-strategy]+ (when-some [{column-name :name+ :keys [base-type]+ :as column}+ (cond+ checkpoint-filter-unique-key+ (source->checkpoint-filter-unique-key query source-incremental-strategy)+ checkpoint-filter+ (when-some [field-id (t2/select-one-pk :model/Field+ :table_id (:id table)+ :name checkpoint-filter)]+ (lib.metadata/field metadata-provider field-id)))]+ (when-not (supported-incremental-filter-type? base-type)+ (throw (ex-info (str "Checkpoint column '" column-name "' has unsupported type " (pr-str base-type) ". "+ "Only numeric and temporal columns are supported for incremental filtering.")+ {:column-name column-name+ :base-type base-type})))+ column)))(defn next-checkpoint"Build a query to compute the MAX of the checkpoint column from the target table.diff --git a/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/api_test.cljindex a7d95408a1d..c346cae39e7 100644--- a/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj@@ -3,6 +3,7 @@(:require[clojure.test :refer :all][medley.core :as m]+ [metabase-enterprise.transforms.api][metabase-enterprise.transforms.query-test-util :as query-test-util][metabase-enterprise.transforms.test-dataset :as transforms-dataset][metabase-enterprise.transforms.test-util :refer [get-test-schema@@ -1032,7 +1033,8 @@(testing "Successfully extracts columns from a simple SELECT query"(let [response (mt/user-http-request :crowberto :post 200 "ee/transform/extract-columns"{:query (make-native-query "SELECT id, name, category, price FROM transforms_products")})]- (is (= ["id" "name" "category" "price"] (:columns response)))))+ (is (= ["id" "price"] (:columns response))+ "Should only return numeric (id, price) columns, filtering out text columns (name, category)")))(testing "Returns nil for invalid SQL"(let [response (mt/user-http-request :crowberto :post 200 "ee/transform/extract-columns"@@ -1042,7 +1044,14 @@(testing "Extracts columns from query with aliases"(let [response (mt/user-http-request :crowberto :post 200 "ee/transform/extract-columns"{:query (make-native-query "SELECT id AS product_id, name AS product_name FROM transforms_products")})]- (is (= ["product_id" "product_name"] (:columns response)))))+ (is (= ["product_id"] (:columns response))+ "Should only return numeric column (id), filtering out text column (name)")))++ (testing "Filters columns by type - only returns numeric and temporal columns"+ (let [response (mt/user-http-request :crowberto :post 200 "ee/transform/extract-columns"+ {:query (make-native-query "SELECT id, name, category, price, created_at FROM transforms_products")})]+ (is (= ["id" "price" "created_at"] (:columns response))+ "Should return numeric (id, price) and temporal (created_at) columns, filtering out text columns (name, category)")))(testing "Requires superuser permissions"(is (= "You don't have permissions to do that."@@ -1175,3 +1184,102 @@(let [items (:data (mt/user-http-request :rasta :get 200(format "collection/%d/items" collection-id)))](is (empty? items))))))))++(deftest native-incremental-column-type-validated-on-create-test+ (testing "POST /api/ee/transform column type validation"+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table ::extract-columns-from-query)+ (mt/with-premium-features #{:transforms}+ (mt/dataset transforms-dataset/transforms-test+ (let [schema (get-test-schema)]+ (testing "Rejects unsupported checkpoint column type (text)"+ (let [response (mt/user-http-request :crowberto :post 400 "ee/transform"+ {:name "Invalid Incremental Transform"+ :source {:type "query"+ :query (lib/native-query (mt/metadata-provider)+ "SELECT id, name, category, price FROM transforms_products")+ :source-incremental-strategy {:type "checkpoint"+ :checkpoint-filter "name"}}+ :target {:type "table-incremental"+ :schema schema+ :name "invalid_incremental"+ :target-incremental-strategy {:type "append"}}})]+ (is (string? response))+ (is (re-find #"Only numeric and temporal" response))))))))))++(deftest native-incremental-column-type-validated-on-update-test+ (testing "PUT /api/ee/transform column type validation"+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table ::extract-columns-from-query)+ (mt/with-premium-features #{:transforms}+ (mt/dataset transforms-dataset/transforms-test+ (let [schema (get-test-schema)]+ (with-transform-cleanup! [table-name "update_incremental_test"]+ (let [;; Create a non-incremental transform first+ created (mt/user-http-request :crowberto :post 200 "ee/transform"+ {:name "Test Transform"+ :source {:type "query"+ :query (lib/native-query (mt/metadata-provider)+ "SELECT id, name, category FROM transforms_products")}+ :target {:type "table"+ :schema schema+ :name table-name}})]+ (testing "Rejects update to unsupported checkpoint column type (text)"+ (let [response (mt/user-http-request :crowberto :put 400+ (format "ee/transform/%d" (:id created))+ {:source {:type "query"+ :query (lib/native-query (mt/metadata-provider)+ "SELECT id, name, category FROM transforms_products")+ :source-incremental-strategy {:type "checkpoint"+ :checkpoint-filter "category"}}+ :target {:type "table-incremental"+ :schema schema+ :name table-name+ :target-incremental-strategy {:type "append"}}})]+ (is (string? response))+ (is (re-find #"Only numeric and temporal" response))))))))))))++(deftest mbql-incremental-column-type-validated-on-create-test+ (testing "MBQL query with checkpoint-filter-unique-key - checkpoint column type validation"+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table ::extract-columns-from-query)+ (mt/with-premium-features #{:transforms}+ (mt/dataset transforms-dataset/transforms-test+ (let [schema (get-test-schema)]+ (testing "Rejects unsupported checkpoint column type (text)"+ (let [query (mt/mbql-query transforms_products)+ response (mt/user-http-request :crowberto :post 400 "ee/transform"+ {:name "Invalid MBQL Incremental"+ :source {:type "query"+ :query query+ :source-incremental-strategy {:type "checkpoint"+ :checkpoint-filter-unique-key "column-unique-key-v1$name"}}+ :target {:type "table-incremental"+ :schema schema+ :name "invalid_mbql_incremental"+ :target-incremental-strategy {:type "append"}}})]+ (is (string? response))+ (is (re-find #"not supported" response))))))))))++(deftest native-incremental-column-validation-when-not-extractable-test+ (testing "Native query checkpoint column validation with text input fallback"+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table ::extract-columns-from-query)+ (mt/with-premium-features #{:transforms}+ (mt/dataset transforms-dataset/transforms-test+ (let [schema (get-test-schema)]+ (testing "Accepts any column if they were not extractable"+ (with-transform-cleanup! [table-name "fallback_test_unextracted"]+ (with-redefs [metabase-enterprise.transforms.api/extract-all-columns-from-query+ ;; simulate lack of driver support for extraction+ (fn [_driver _database-id _query] nil)]+ (let [response (mt/user-http-request :crowberto :post 200 "ee/transform"+ {:name "Unextracted Column - Text Input Fallback"+ :source {:type "query"+ :query (lib/native-query (mt/metadata-provider)+ "SELECT id, name, created_at FROM transforms_products")+ :source-incremental-strategy {:type "checkpoint"+ ;; created_at is in the query but not in our stubbed extraction+ :checkpoint-filter "created_at"}}+ :target {:type "table-incremental"+ :schema schema+ :name table-name+ :target-incremental-strategy {:type "append"}}})]+ (is (some? (:id response))+ "Should accept column not in extracted metadata, allowing text input fallback")))))))))))diff --git a/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.cljindex b9cf55a5647..f750364ce57 100644--- a/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj@@ -481,6 +481,29 @@(is (= 18 row-count) "Should append 2 new rows (16 + 2 = 18)")(is (some? checkpoint) "Checkpoint should be updated")))))))))))))))+(deftest unsupported-checkpoint-column-type-test+ (testing "Transform fails at runtime with unsupported checkpoint column type"+ (mt/test-drivers (test-drivers)+ (mt/with-premium-features #{:transforms}+ (mt/dataset transforms-dataset/transforms-test+ (with-transform-cleanup! [target-table "unsupported_type_test"]+ (let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))+ ;; Create transform with text column (unsupported type) as checkpoint+ transform-payload {:name "Invalid Checkpoint Type Transform"+ :source {:type "query"+ :query (make-incremental-source-query-without-template-tag schema)+ :source-incremental-strategy {:type "checkpoint"+ :checkpoint-filter "name"}}+ :target {:type "table-incremental"+ :schema schema+ :name target-table+ :database (mt/id)+ :target-incremental-strategy {:type "append"}}}]+ (testing "API validation rejects unsupported checkpoint column type"+ (let [response (mt/user-http-request :crowberto :post 400 "ee/transform" transform-payload)]+ (is (string? response))+ (is (re-find #"unsupported type" response)))))))))))+(deftest ^:postgres-only native-query-with-temporal-checkpoint-test(testing "Native query with temporal checkpoint";; we test only in postgres because it's easy to cast to ::timestamp