Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Revert the revert of column validation for incremental transforms (#68460)

Dan Stonea2c4eef9dece
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/api.clj b/enterprise/backend/src/metabase_enterprise/transforms/api.clj
index 3e6c465a616..cd71361aa82 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/api.clj
@@ -20,6 +20,7 @@
[metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
[metabase.driver.util :as driver.u]
[metabase.events.core :as events]
+ [metabase.lib.core :as lib]
[metabase.query-processor.compile :as qp.compile]
[metabase.query-processor.schema :as qp.schema]
[metabase.request.core :as request]
@@ -122,6 +123,83 @@
[:tag_ids {:optional true} [:maybe (ms/QueryVectorOf ms/IntGreaterThanOrEqualToZero)]]]]
(get-transforms query-params))
+(defn- extract-all-columns-from-query
+ "Extracts column metadata (name and type) from a query.
+
+ Returns a sequence of maps with `:name` and `:base_type` keys, or nil if extraction fails.
+
+ The query is first compiled to native SQL, then uses PreparedStatement.getMetaData()
+ to inspect the query structure. This works for most modern JDBC drivers but may not
+ be supported by all drivers or for all query types."
+ [driver database-id query]
+ (try
+ (let [{:keys [query]} (qp.compile/compile query)]
+ (sql-jdbc.execute/do-with-connection-with-options
+ driver
+ database-id
+ {}
+ (fn [conn]
+ (with-open [^PreparedStatement stmt (sql-jdbc.execute/prepared-statement driver conn query [])]
+ (when-let [rsmeta (.getMetaData stmt)]
+ (seq (sql-jdbc.execute/column-metadata driver rsmeta)))))))
+ (catch Exception e
+ (log/debugf e "Failed to extract columns from query: %s" (ex-message e))
+ nil)))
+
+(defn- extract-incremental-filter-columns-from-query
+ "Extracts column names suitable for incremental transform checkpoint filtering.
+
+ This function is specifically for incremental transform checkpoint column selection.
+ It only returns columns with types supported for checkpoint filtering:
+ - Temporal types (timestamp, timestamp with timezone)
+ - Numeric types (integer, float, decimal)
+
+ Text, boolean, and other types are filtered out as they are not supported for
+ incremental checkpointing.
+
+ Returns a vector of column names (as strings), or nil if extraction fails.
+
+ The query is first compiled to native SQL, then uses PreparedStatement.getMetaData()
+ to inspect the query structure. This works for most modern JDBC drivers but may not
+ be supported by all drivers or for all query types."
+ [driver database-id query]
+ (some->> (extract-all-columns-from-query driver database-id query)
+ (filter (comp transforms.util/supported-incremental-filter-type? :base_type))
+ (mapv :name)))
+
+(defn- validate-incremental-column-type!
+ "Validates that the checkpoint column for an incremental transform has a supported type.
+
+ For MBQL/Python transforms, resolves the column from the query using the unique key.
+ For native queries, extracts columns from the query and checks the checkpoint-filter column.
+
+ Throws a 400 error if the column type is not supported or cannot be resolved."
+ [{:keys [source]}]
+ (when-let [{:keys [checkpoint-filter checkpoint-filter-unique-key] strategy-type :type}
+ (:source-incremental-strategy source)]
+ (when (and (= :query (:type source)) (= "checkpoint" strategy-type))
+ (let [{:keys [query]} source
+ database-id (:database query)
+ database (api/check-404 (t2/select-one :model/Database :id database-id))
+ driver-name (driver/the-initialized-driver (:engine database))]
+ (cond
+ ;; For MBQL, resolve column from query metadata
+ checkpoint-filter-unique-key
+ (let [column (lib/column-with-unique-key query checkpoint-filter-unique-key)]
+ (api/check-400 column (deferred-tru "Checkpoint column not found in query."))
+ (api/check-400 (transforms.util/supported-incremental-filter-type? (:base-type column))
+ (deferred-tru "Checkpoint column type {0} is not supported. Only numeric and temporal types are supported for incremental filtering."
+ (pr-str (:base-type column)))))
+
+ ;; For native query with checkpoint-filter, validate type if we can extract the column metadata
+ checkpoint-filter
+ (when-some [column-metadata (seq (extract-all-columns-from-query driver-name database-id query))]
+ (when-some [column (first (filter #(= checkpoint-filter (:name %)) column-metadata))]
+ (api/check-400 (transforms.util/supported-incremental-filter-type? (:base_type column))
+ (deferred-tru "Checkpoint column ''{0}'' has unsupported type {1}. Only numeric and temporal columns are supported for incremental filtering."
+ checkpoint-filter
+ (pr-str (:base_type column)))))))))))
+
;; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to
;; use our API + we will need it when we make auto-TypeScript-signature generation happen
;;
@@ -143,6 +221,7 @@
(api/create-check :model/Transform body)
(check-database-feature body)
(check-feature-enabled! body)
+ (validate-incremental-column-type! body)
(api/check (not (transforms.util/target-table-exists? body))
403
@@ -256,6 +335,7 @@
;; we must validate on a full transform object
(check-feature-enabled! new)
(check-database-feature new)
+ (validate-incremental-column-type! new)
(when (transforms.util/query-transform? old)
(when-let [{:keys [cycle-str]} (transforms.ordering/get-transform-cycle new)]
(throw (ex-info (str "Cyclic transform definitions detected: " cycle-str)
@@ -338,30 +418,6 @@
:run_id run-id})
(assoc :status 202)))))
-(defn- extract-columns-from-query
- "Attempts to extract column names from an MBQL query without executing it.
-
- Returns a vector of column names (as strings), or nil if extraction fails.
-
- The query is first compiled to native SQL, hen uses PreparedStatement.getMetaData()
- to inspect the query structure. This works for most modern JDBC drivers but may not
- be supported by all drivers or for all query types."
- [driver database-id query]
- (try
- (let [{:keys [query]} (qp.compile/compile query)]
- (sql-jdbc.execute/do-with-connection-with-options
- driver
- database-id
- {}
- (fn [conn]
- (with-open [^PreparedStatement stmt (sql-jdbc.execute/prepared-statement driver conn query [])]
- (when-let [rsmeta (.getMetaData stmt)]
- (let [columns (sql-jdbc.execute/column-metadata driver rsmeta)]
- (seq (mapv :name columns))))))))
- (catch Exception e
- (log/debugf e "Failed to extract columns from query: %s" (ex-message e))
- nil)))
-
(defn- simple-native-query?
"Checks if a native SQL query string is simple enough for automatic checkpoint insertion."
[sql-string]
@@ -402,16 +458,19 @@
(api/check-superuser)
(simple-native-query? query))
-;; TODO (Cam 2025-12-04) please add a response schema to this API endpoint, it makes it easier for our customers to
-;; use our API + we will need it when we make auto-TypeScript-signature generation happen
-;;
-#_{:clj-kondo/ignore [:metabase/validate-defendpoint-has-response-schema]}
(api.macros/defendpoint :post "/extract-columns"
- "Extract column names from an MBQL query without executing it.
+ :- [:map [:columns [:maybe [:sequential :string]]]]
+ "Extract column names suitable for incremental transform checkpoint filtering.
+
+ This endpoint is specifically for populating the checkpoint column dropdown in
+ incremental transforms. It only returns columns with types supported for checkpoint
+ filtering: temporal (timestamp/tz) and numeric (int/float) types.
+
+ Text, boolean, and other unsupported column types are filtered out.
- Compiles the query to native SQL using [[qp.compile/compile-with-inline-parameters]],
+ The query is compiled to native SQL using [[qp.compile/compile-with-inline-parameters]],
which handles parameterized queries with template tags. Then extracts column names
- using PreparedStatement metadata.
+ and types using PreparedStatement metadata.
Returns a map with a :columns key containing a vector of column names (strings).
If extraction fails, returns nil for :columns."
@@ -421,9 +480,9 @@
[:query ::qp.schema/any-query]]]
(api/check-superuser)
(let [database-id (:database query)
- database (api/check-404 (t2/select-one :model/Database :id database-id))
+ database (api/check-404 (t2/select-one :model/Database :id database-id))
driver-name (driver/the-initialized-driver (:engine database))
- columns (extract-columns-from-query driver-name database-id query)]
+ columns (extract-incremental-filter-columns-from-query driver-name database-id query)]
{:columns columns}))
(def ^{:arglists '([request respond raise])} routes
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/util.clj b/enterprise/backend/src/metabase_enterprise/transforms/util.clj
index 375cf0e8f98..068fdf7cd0a 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/util.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/util.clj
@@ -214,6 +214,14 @@
[source]
(= :checkpoint (some-> source :source-incremental-strategy :type keyword)))
+(defn supported-incremental-filter-type?
+ "Returns true if the given base-type is supported for incremental filtering.
+
+ We only support temporal (timestamp/tz) and numeric (int/float) types."
+ [base-type]
+ (or (isa? base-type :type/Temporal)
+ (isa? base-type :type/Number)))
+
(defn- source->checkpoint-filter-unique-key
"Extract the checkpoint filter column from `query` using the unique key specified in `source-incremental-strategy`."
[query source-incremental-strategy]
@@ -223,15 +231,29 @@
"Resolve the checkpoint filter column for an incremental transform.
Tries to resolve the column using the unique key first.
- Falls back to looking up the column by name from the target table if a `:checkpoint-filter` is specified."
+ Falls back to looking up the column by name from the target table if a `:checkpoint-filter` is specified.
+
+ Validates that the resolved column has a supported type for checkpoint filtering (numeric or temporal).
+ Throws an exception if the column type is not supported."
[query source-incremental-strategy table metadata-provider]
- (or
- (source->checkpoint-filter-unique-key query source-incremental-strategy)
- (when-let [field-name (-> source-incremental-strategy :checkpoint-filter)]
- (when-let [field-id (t2/select-one-pk :model/Field
- :table_id (:id table)
- :name field-name)]
- (lib.metadata/field metadata-provider field-id)))))
+ (let [{:keys [checkpoint-filter checkpoint-filter-unique-key]} source-incremental-strategy]
+ (when-some [{column-name :name
+ :keys [base-type]
+ :as column}
+ (cond
+ checkpoint-filter-unique-key
+ (source->checkpoint-filter-unique-key query source-incremental-strategy)
+ checkpoint-filter
+ (when-some [field-id (t2/select-one-pk :model/Field
+ :table_id (:id table)
+ :name checkpoint-filter)]
+ (lib.metadata/field metadata-provider field-id)))]
+ (when-not (supported-incremental-filter-type? base-type)
+ (throw (ex-info (str "Checkpoint column '" column-name "' has unsupported type " (pr-str base-type) ". "
+ "Only numeric and temporal columns are supported for incremental filtering.")
+ {:column-name column-name
+ :base-type base-type})))
+ column)))
(defn next-checkpoint
"Build a query to compute the MAX of the checkpoint column from the target table.
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj
index a7d95408a1d..c346cae39e7 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj
@@ -3,6 +3,7 @@
(:require
[clojure.test :refer :all]
[medley.core :as m]
+ [metabase-enterprise.transforms.api]
[metabase-enterprise.transforms.query-test-util :as query-test-util]
[metabase-enterprise.transforms.test-dataset :as transforms-dataset]
[metabase-enterprise.transforms.test-util :refer [get-test-schema
@@ -1032,7 +1033,8 @@
(testing "Successfully extracts columns from a simple SELECT query"
(let [response (mt/user-http-request :crowberto :post 200 "ee/transform/extract-columns"
{:query (make-native-query "SELECT id, name, category, price FROM transforms_products")})]
- (is (= ["id" "name" "category" "price"] (:columns response)))))
+ (is (= ["id" "price"] (:columns response))
+ "Should only return numeric (id, price) columns, filtering out text columns (name, category)")))
(testing "Returns nil for invalid SQL"
(let [response (mt/user-http-request :crowberto :post 200 "ee/transform/extract-columns"
@@ -1042,7 +1044,14 @@
(testing "Extracts columns from query with aliases"
(let [response (mt/user-http-request :crowberto :post 200 "ee/transform/extract-columns"
{:query (make-native-query "SELECT id AS product_id, name AS product_name FROM transforms_products")})]
- (is (= ["product_id" "product_name"] (:columns response)))))
+ (is (= ["product_id"] (:columns response))
+ "Should only return numeric column (id), filtering out text column (name)")))
+
+ (testing "Filters columns by type - only returns numeric and temporal columns"
+ (let [response (mt/user-http-request :crowberto :post 200 "ee/transform/extract-columns"
+ {:query (make-native-query "SELECT id, name, category, price, created_at FROM transforms_products")})]
+ (is (= ["id" "price" "created_at"] (:columns response))
+ "Should return numeric (id, price) and temporal (created_at) columns, filtering out text columns (name, category)")))
(testing "Requires superuser permissions"
(is (= "You don't have permissions to do that."
@@ -1175,3 +1184,102 @@
(let [items (:data (mt/user-http-request :rasta :get 200
(format "collection/%d/items" collection-id)))]
(is (empty? items))))))))
+
+(deftest native-incremental-column-type-validated-on-create-test
+ (testing "POST /api/ee/transform column type validation"
+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table ::extract-columns-from-query)
+ (mt/with-premium-features #{:transforms}
+ (mt/dataset transforms-dataset/transforms-test
+ (let [schema (get-test-schema)]
+ (testing "Rejects unsupported checkpoint column type (text)"
+ (let [response (mt/user-http-request :crowberto :post 400 "ee/transform"
+ {:name "Invalid Incremental Transform"
+ :source {:type "query"
+ :query (lib/native-query (mt/metadata-provider)
+ "SELECT id, name, category, price FROM transforms_products")
+ :source-incremental-strategy {:type "checkpoint"
+ :checkpoint-filter "name"}}
+ :target {:type "table-incremental"
+ :schema schema
+ :name "invalid_incremental"
+ :target-incremental-strategy {:type "append"}}})]
+ (is (string? response))
+ (is (re-find #"Only numeric and temporal" response))))))))))
+
+(deftest native-incremental-column-type-validated-on-update-test
+ (testing "PUT /api/ee/transform column type validation"
+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table ::extract-columns-from-query)
+ (mt/with-premium-features #{:transforms}
+ (mt/dataset transforms-dataset/transforms-test
+ (let [schema (get-test-schema)]
+ (with-transform-cleanup! [table-name "update_incremental_test"]
+ (let [;; Create a non-incremental transform first
+ created (mt/user-http-request :crowberto :post 200 "ee/transform"
+ {:name "Test Transform"
+ :source {:type "query"
+ :query (lib/native-query (mt/metadata-provider)
+ "SELECT id, name, category FROM transforms_products")}
+ :target {:type "table"
+ :schema schema
+ :name table-name}})]
+ (testing "Rejects update to unsupported checkpoint column type (text)"
+ (let [response (mt/user-http-request :crowberto :put 400
+ (format "ee/transform/%d" (:id created))
+ {:source {:type "query"
+ :query (lib/native-query (mt/metadata-provider)
+ "SELECT id, name, category FROM transforms_products")
+ :source-incremental-strategy {:type "checkpoint"
+ :checkpoint-filter "category"}}
+ :target {:type "table-incremental"
+ :schema schema
+ :name table-name
+ :target-incremental-strategy {:type "append"}}})]
+ (is (string? response))
+ (is (re-find #"Only numeric and temporal" response))))))))))))
+
+(deftest mbql-incremental-column-type-validated-on-create-test
+ (testing "MBQL query with checkpoint-filter-unique-key - checkpoint column type validation"
+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table ::extract-columns-from-query)
+ (mt/with-premium-features #{:transforms}
+ (mt/dataset transforms-dataset/transforms-test
+ (let [schema (get-test-schema)]
+ (testing "Rejects unsupported checkpoint column type (text)"
+ (let [query (mt/mbql-query transforms_products)
+ response (mt/user-http-request :crowberto :post 400 "ee/transform"
+ {:name "Invalid MBQL Incremental"
+ :source {:type "query"
+ :query query
+ :source-incremental-strategy {:type "checkpoint"
+ :checkpoint-filter-unique-key "column-unique-key-v1$name"}}
+ :target {:type "table-incremental"
+ :schema schema
+ :name "invalid_mbql_incremental"
+ :target-incremental-strategy {:type "append"}}})]
+ (is (string? response))
+ (is (re-find #"not supported" response))))))))))
+
+(deftest native-incremental-column-validation-when-not-extractable-test
+ (testing "Native query checkpoint column validation with text input fallback"
+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table ::extract-columns-from-query)
+ (mt/with-premium-features #{:transforms}
+ (mt/dataset transforms-dataset/transforms-test
+ (let [schema (get-test-schema)]
+ (testing "Accepts any column if they were not extractable"
+ (with-transform-cleanup! [table-name "fallback_test_unextracted"]
+ (with-redefs [metabase-enterprise.transforms.api/extract-all-columns-from-query
+ ;; simulate lack of driver support for extraction
+ (fn [_driver _database-id _query] nil)]
+ (let [response (mt/user-http-request :crowberto :post 200 "ee/transform"
+ {:name "Unextracted Column - Text Input Fallback"
+ :source {:type "query"
+ :query (lib/native-query (mt/metadata-provider)
+ "SELECT id, name, created_at FROM transforms_products")
+ :source-incremental-strategy {:type "checkpoint"
+ ;; created_at is in the query but not in our stubbed extraction
+ :checkpoint-filter "created_at"}}
+ :target {:type "table-incremental"
+ :schema schema
+ :name table-name
+ :target-incremental-strategy {:type "append"}}})]
+ (is (some? (:id response))
+ "Should accept column not in extracted metadata, allowing text input fallback")))))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj
index b9cf55a5647..f750364ce57 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj
@@ -481,6 +481,29 @@
(is (= 18 row-count) "Should append 2 new rows (16 + 2 = 18)")
(is (some? checkpoint) "Checkpoint should be updated")))))))))))))))
+(deftest unsupported-checkpoint-column-type-test
+ (testing "Transform fails at runtime with unsupported checkpoint column type"
+ (mt/test-drivers (test-drivers)
+ (mt/with-premium-features #{:transforms}
+ (mt/dataset transforms-dataset/transforms-test
+ (with-transform-cleanup! [target-table "unsupported_type_test"]
+ (let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))
+ ;; Create transform with text column (unsupported type) as checkpoint
+ transform-payload {:name "Invalid Checkpoint Type Transform"
+ :source {:type "query"
+ :query (make-incremental-source-query-without-template-tag schema)
+ :source-incremental-strategy {:type "checkpoint"
+ :checkpoint-filter "name"}}
+ :target {:type "table-incremental"
+ :schema schema
+ :name target-table
+ :database (mt/id)
+ :target-incremental-strategy {:type "append"}}}]
+ (testing "API validation rejects unsupported checkpoint column type"
+ (let [response (mt/user-http-request :crowberto :post 400 "ee/transform" transform-payload)]
+ (is (string? response))
+ (is (re-find #"unsupported type" response)))))))))))
+
(deftest ^:postgres-only native-query-with-temporal-checkpoint-test
(testing "Native query with temporal checkpoint"
;; we test only in postgres because it's easy to cast to ::timestamp