The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Revert transforms changes recently (#68446)
dpsutton410471a4d024
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/api.clj b/enterprise/backend/src/metabase_enterprise/transforms/api.cljindex b6fed336758..e100e92e66d 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/api.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/api.clj@@ -20,7 +20,6 @@[metabase.driver.sql-jdbc.execute :as sql-jdbc.execute][metabase.driver.util :as driver.u][metabase.events.core :as events]- [metabase.lib.core :as lib][metabase.query-processor.compile :as qp.compile][metabase.query-processor.schema :as qp.schema][metabase.request.core :as request]@@ -123,83 +122,6 @@[:tag_ids {:optional true} [:maybe (ms/QueryVectorOf ms/IntGreaterThanOrEqualToZero)]]]](get-transforms query-params))-(defn- extract-all-columns-from-query- "Extracts column metadata (name and type) from a query.-- Returns a sequence of maps with `:name` and `:base_type` keys, or nil if extraction fails.-- The query is first compiled to native SQL, then uses PreparedStatement.getMetaData()- to inspect the query structure. This works for most modern JDBC drivers but may not- be supported by all drivers or for all query types."- [driver database-id query]- (try- (let [{:keys [query]} (qp.compile/compile query)]- (sql-jdbc.execute/do-with-connection-with-options- driver- database-id- {}- (fn [conn]- (with-open [^PreparedStatement stmt (sql-jdbc.execute/prepared-statement driver conn query [])]- (when-let [rsmeta (.getMetaData stmt)]- (seq (sql-jdbc.execute/column-metadata driver rsmeta)))))))- (catch Exception e- (log/debugf e "Failed to extract columns from query: %s" (ex-message e))- nil)))--(defn- extract-incremental-filter-columns-from-query- "Extracts column names suitable for incremental transform checkpoint filtering.-- This function is specifically for incremental transform checkpoint column selection.- It only returns columns with types supported for checkpoint filtering:- - Temporal types (timestamp, timestamp with timezone)- - Numeric types (integer, float, decimal)-- Text, boolean, and other types are filtered out as they are not supported for- incremental checkpointing.-- Returns a vector of column names (as strings), or nil if extraction fails.-- The query is first compiled to native SQL, then uses PreparedStatement.getMetaData()- to inspect the query structure. This works for most modern JDBC drivers but may not- be supported by all drivers or for all query types."- [driver database-id query]- (some->> (extract-all-columns-from-query driver database-id query)- (filter (comp transforms.util/supported-incremental-filter-type? :base_type))- (mapv :name)))--(defn- validate-incremental-column-type!- "Validates that the checkpoint column for an incremental transform has a supported type.-- For MBQL/Python transforms, resolves the column from the query using the unique key.- For native queries, extracts columns from the query and checks the checkpoint-filter column.-- Throws a 400 error if the column type is not supported or cannot be resolved."- [{:keys [source]}]- (when-let [{:keys [checkpoint-filter checkpoint-filter-unique-key] strategy-type :type}- (:source-incremental-strategy source)]- (when (= "checkpoint" strategy-type)- (let [{:keys [query]} source- database-id (:database query)- database (api/check-404 (t2/select-one :model/Database :id database-id))- driver-name (driver/the-initialized-driver (:engine database))]- (cond- ;; For MBQL/Python with unique key, resolve column from query metadata- checkpoint-filter-unique-key- (let [column (lib/column-with-unique-key query checkpoint-filter-unique-key)]- (api/check-400 column (deferred-tru "Checkpoint column not found in query."))- (api/check-400 (transforms.util/supported-incremental-filter-type? (:base-type column))- (deferred-tru "Checkpoint column type {0} is not supported. Only numeric and temporal types are supported for incremental filtering."- (pr-str (:base-type column)))))-- ;; For native query with checkpoint-filter, validate type if we can extract the column metadata- checkpoint-filter- (when-some [column-metadata (seq (extract-all-columns-from-query driver-name database-id query))]- (when-some [column (first (filter #(= checkpoint-filter (:name %)) column-metadata))]- (api/check-400 (transforms.util/supported-incremental-filter-type? (:base_type column))- (deferred-tru "Checkpoint column ''{0}'' has unsupported type {1}. Only numeric and temporal columns are supported for incremental filtering."- checkpoint-filter- (pr-str (:base_type column)))))))))))-;; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to;; use our API + we will need it when we make auto-TypeScript-signature generation happen;;@@ -221,7 +143,6 @@(api/check-superuser)(check-database-feature body)(check-feature-enabled! body)- (validate-incremental-column-type! body)(api/check (not (transforms.util/target-table-exists? body))403@@ -337,7 +258,6 @@;; we must validate on a full transform object(check-feature-enabled! new)(check-database-feature new)- (validate-incremental-column-type! new)(when (transforms.util/query-transform? old)(when-let [{:keys [cycle-str]} (transforms.ordering/get-transform-cycle new)](throw (ex-info (str "Cyclic transform definitions detected: " cycle-str)@@ -423,6 +343,30 @@:run_id run-id})(assoc :status 202)))))+(defn- extract-columns-from-query+ "Attempts to extract column names from an MBQL query without executing it.++ Returns a vector of column names (as strings), or nil if extraction fails.++ The query is first compiled to native SQL, hen uses PreparedStatement.getMetaData()+ to inspect the query structure. This works for most modern JDBC drivers but may not+ be supported by all drivers or for all query types."+ [driver database-id query]+ (try+ (let [{:keys [query]} (qp.compile/compile query)]+ (sql-jdbc.execute/do-with-connection-with-options+ driver+ database-id+ {}+ (fn [conn]+ (with-open [^PreparedStatement stmt (sql-jdbc.execute/prepared-statement driver conn query [])]+ (when-let [rsmeta (.getMetaData stmt)]+ (let [columns (sql-jdbc.execute/column-metadata driver rsmeta)]+ (seq (mapv :name columns))))))))+ (catch Exception e+ (log/debugf e "Failed to extract columns from query: %s" (ex-message e))+ nil)))+(defn- simple-native-query?"Checks if a native SQL query string is simple enough for automatic checkpoint insertion."[sql-string]@@ -463,19 +407,16 @@(api/check-superuser)(simple-native-query? query))+;; TODO (Cam 2025-12-04) please add a response schema to this API endpoint, it makes it easier for our customers to+;; use our API + we will need it when we make auto-TypeScript-signature generation happen+;;+#_{:clj-kondo/ignore [:metabase/validate-defendpoint-has-response-schema]}(api.macros/defendpoint :post "/extract-columns"- :- [:map [:columns [:maybe [:sequential :string]]]]- "Extract column names suitable for incremental transform checkpoint filtering.-- This endpoint is specifically for populating the checkpoint column dropdown in- incremental transforms. It only returns columns with types supported for checkpoint- filtering: temporal (timestamp/tz) and numeric (int/float) types.-- Text, boolean, and other unsupported column types are filtered out.+ "Extract column names from an MBQL query without executing it.- The query is compiled to native SQL using [[qp.compile/compile-with-inline-parameters]],+ Compiles the query to native SQL using [[qp.compile/compile-with-inline-parameters]],which handles parameterized queries with template tags. Then extracts column names- and types using PreparedStatement metadata.+ using PreparedStatement metadata.Returns a map with a :columns key containing a vector of column names (strings).If extraction fails, returns nil for :columns."@@ -485,9 +426,9 @@[:query ::qp.schema/any-query]]](api/check-superuser)(let [database-id (:database query)- database (api/check-404 (t2/select-one :model/Database :id database-id))+ database (api/check-404 (t2/select-one :model/Database :id database-id))driver-name (driver/the-initialized-driver (:engine database))- columns (extract-incremental-filter-columns-from-query driver-name database-id query)]+ columns (extract-columns-from-query driver-name database-id query)]{:columns columns}))(def ^{:arglists '([request respond raise])} routesdiff --git a/enterprise/backend/src/metabase_enterprise/transforms/util.clj b/enterprise/backend/src/metabase_enterprise/transforms/util.cljindex fabbb72b355..0891687d0fa 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/util.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/util.clj@@ -214,14 +214,6 @@[source](= :checkpoint (some-> source :source-incremental-strategy :type keyword)))-(defn supported-incremental-filter-type?- "Returns true if the given base-type is supported for incremental filtering.-- We only support temporal (timestamp/tz) and numeric (int/float) types."- [base-type]- (or (isa? base-type :type/Temporal)- (isa? base-type :type/Number)))-(defn- source->checkpoint-filter-unique-key"Extract the checkpoint filter column from `query` using the unique key specified in `source-incremental-strategy`."[query source-incremental-strategy]@@ -231,29 +223,15 @@"Resolve the checkpoint filter column for an incremental transform.Tries to resolve the column using the unique key first.- Falls back to looking up the column by name from the target table if a `:checkpoint-filter` is specified.-- Validates that the resolved column has a supported type for checkpoint filtering (numeric or temporal).- Throws an exception if the column type is not supported."+ Falls back to looking up the column by name from the target table if a `:checkpoint-filter` is specified."[query source-incremental-strategy table metadata-provider]- (let [{:keys [checkpoint-filter checkpoint-filter-unique-key]} source-incremental-strategy]- (when-some [{column-name :name- :keys [base-type]- :as column}- (cond- checkpoint-filter-unique-key- (source->checkpoint-filter-unique-key query source-incremental-strategy)- checkpoint-filter- (when-some [field-id (t2/select-one-pk :model/Field- :table_id (:id table)- :name checkpoint-filter)]- (lib.metadata/field metadata-provider field-id)))]- (when-not (supported-incremental-filter-type? base-type)- (throw (ex-info (str "Checkpoint column '" column-name "' has unsupported type " (pr-str base-type) ". "- "Only numeric and temporal columns are supported for incremental filtering.")- {:column-name column-name- :base-type base-type})))- column)))+ (or+ (source->checkpoint-filter-unique-key query source-incremental-strategy)+ (when-let [field-name (-> source-incremental-strategy :checkpoint-filter)]+ (when-let [field-id (t2/select-one-pk :model/Field+ :table_id (:id table)+ :name field-name)]+ (lib.metadata/field metadata-provider field-id)))))(defn next-checkpoint"Build a query to compute the MAX of the checkpoint column from the target table.diff --git a/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/api_test.cljindex c346cae39e7..a7d95408a1d 100644--- a/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj@@ -3,7 +3,6 @@(:require[clojure.test :refer :all][medley.core :as m]- [metabase-enterprise.transforms.api][metabase-enterprise.transforms.query-test-util :as query-test-util][metabase-enterprise.transforms.test-dataset :as transforms-dataset][metabase-enterprise.transforms.test-util :refer [get-test-schema@@ -1033,8 +1032,7 @@(testing "Successfully extracts columns from a simple SELECT query"(let [response (mt/user-http-request :crowberto :post 200 "ee/transform/extract-columns"{:query (make-native-query "SELECT id, name, category, price FROM transforms_products")})]- (is (= ["id" "price"] (:columns response))- "Should only return numeric (id, price) columns, filtering out text columns (name, category)")))+ (is (= ["id" "name" "category" "price"] (:columns response)))))(testing "Returns nil for invalid SQL"(let [response (mt/user-http-request :crowberto :post 200 "ee/transform/extract-columns"@@ -1044,14 +1042,7 @@(testing "Extracts columns from query with aliases"(let [response (mt/user-http-request :crowberto :post 200 "ee/transform/extract-columns"{:query (make-native-query "SELECT id AS product_id, name AS product_name FROM transforms_products")})]- (is (= ["product_id"] (:columns response))- "Should only return numeric column (id), filtering out text column (name)")))-- (testing "Filters columns by type - only returns numeric and temporal columns"- (let [response (mt/user-http-request :crowberto :post 200 "ee/transform/extract-columns"- {:query (make-native-query "SELECT id, name, category, price, created_at FROM transforms_products")})]- (is (= ["id" "price" "created_at"] (:columns response))- "Should return numeric (id, price) and temporal (created_at) columns, filtering out text columns (name, category)")))+ (is (= ["product_id" "product_name"] (:columns response)))))(testing "Requires superuser permissions"(is (= "You don't have permissions to do that."@@ -1184,102 +1175,3 @@(let [items (:data (mt/user-http-request :rasta :get 200(format "collection/%d/items" collection-id)))](is (empty? items))))))))--(deftest native-incremental-column-type-validated-on-create-test- (testing "POST /api/ee/transform column type validation"- (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table ::extract-columns-from-query)- (mt/with-premium-features #{:transforms}- (mt/dataset transforms-dataset/transforms-test- (let [schema (get-test-schema)]- (testing "Rejects unsupported checkpoint column type (text)"- (let [response (mt/user-http-request :crowberto :post 400 "ee/transform"- {:name "Invalid Incremental Transform"- :source {:type "query"- :query (lib/native-query (mt/metadata-provider)- "SELECT id, name, category, price FROM transforms_products")- :source-incremental-strategy {:type "checkpoint"- :checkpoint-filter "name"}}- :target {:type "table-incremental"- :schema schema- :name "invalid_incremental"- :target-incremental-strategy {:type "append"}}})]- (is (string? response))- (is (re-find #"Only numeric and temporal" response))))))))))--(deftest native-incremental-column-type-validated-on-update-test- (testing "PUT /api/ee/transform column type validation"- (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table ::extract-columns-from-query)- (mt/with-premium-features #{:transforms}- (mt/dataset transforms-dataset/transforms-test- (let [schema (get-test-schema)]- (with-transform-cleanup! [table-name "update_incremental_test"]- (let [;; Create a non-incremental transform first- created (mt/user-http-request :crowberto :post 200 "ee/transform"- {:name "Test Transform"- :source {:type "query"- :query (lib/native-query (mt/metadata-provider)- "SELECT id, name, category FROM transforms_products")}- :target {:type "table"- :schema schema- :name table-name}})]- (testing "Rejects update to unsupported checkpoint column type (text)"- (let [response (mt/user-http-request :crowberto :put 400- (format "ee/transform/%d" (:id created))- {:source {:type "query"- :query (lib/native-query (mt/metadata-provider)- "SELECT id, name, category FROM transforms_products")- :source-incremental-strategy {:type "checkpoint"- :checkpoint-filter "category"}}- :target {:type "table-incremental"- :schema schema- :name table-name- :target-incremental-strategy {:type "append"}}})]- (is (string? response))- (is (re-find #"Only numeric and temporal" response))))))))))))--(deftest mbql-incremental-column-type-validated-on-create-test- (testing "MBQL query with checkpoint-filter-unique-key - checkpoint column type validation"- (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table ::extract-columns-from-query)- (mt/with-premium-features #{:transforms}- (mt/dataset transforms-dataset/transforms-test- (let [schema (get-test-schema)]- (testing "Rejects unsupported checkpoint column type (text)"- (let [query (mt/mbql-query transforms_products)- response (mt/user-http-request :crowberto :post 400 "ee/transform"- {:name "Invalid MBQL Incremental"- :source {:type "query"- :query query- :source-incremental-strategy {:type "checkpoint"- :checkpoint-filter-unique-key "column-unique-key-v1$name"}}- :target {:type "table-incremental"- :schema schema- :name "invalid_mbql_incremental"- :target-incremental-strategy {:type "append"}}})]- (is (string? response))- (is (re-find #"not supported" response))))))))))--(deftest native-incremental-column-validation-when-not-extractable-test- (testing "Native query checkpoint column validation with text input fallback"- (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table ::extract-columns-from-query)- (mt/with-premium-features #{:transforms}- (mt/dataset transforms-dataset/transforms-test- (let [schema (get-test-schema)]- (testing "Accepts any column if they were not extractable"- (with-transform-cleanup! [table-name "fallback_test_unextracted"]- (with-redefs [metabase-enterprise.transforms.api/extract-all-columns-from-query- ;; simulate lack of driver support for extraction- (fn [_driver _database-id _query] nil)]- (let [response (mt/user-http-request :crowberto :post 200 "ee/transform"- {:name "Unextracted Column - Text Input Fallback"- :source {:type "query"- :query (lib/native-query (mt/metadata-provider)- "SELECT id, name, created_at FROM transforms_products")- :source-incremental-strategy {:type "checkpoint"- ;; created_at is in the query but not in our stubbed extraction- :checkpoint-filter "created_at"}}- :target {:type "table-incremental"- :schema schema- :name table-name- :target-incremental-strategy {:type "append"}}})]- (is (some? (:id response))- "Should accept column not in extracted metadata, allowing text input fallback")))))))))))diff --git a/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.cljindex 8801067128c..8f32f1a2262 100644--- a/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj@@ -481,29 +481,6 @@(is (= 18 row-count) "Should append 2 new rows (16 + 2 = 18)")(is (some? checkpoint) "Checkpoint should be updated")))))))))))))))-(deftest unsupported-checkpoint-column-type-test- (testing "Transform fails at runtime with unsupported checkpoint column type"- (mt/test-drivers (test-drivers)- (mt/with-premium-features #{:transforms}- (mt/dataset transforms-dataset/transforms-test- (with-transform-cleanup! [target-table "unsupported_type_test"]- (let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))- ;; Create transform with text column (unsupported type) as checkpoint- transform-payload {:name "Invalid Checkpoint Type Transform"- :source {:type "query"- :query (make-incremental-source-query-without-template-tag schema)- :source-incremental-strategy {:type "checkpoint"- :checkpoint-filter "name"}}- :target {:type "table-incremental"- :schema schema- :name target-table- :database (mt/id)- :target-incremental-strategy {:type "append"}}}]- (testing "API validation rejects unsupported checkpoint column type"- (let [response (mt/user-http-request :crowberto :post 400 "ee/transform" transform-payload)]- (is (string? response))- (is (re-find #"not supported" response)))))))))))-(deftest ^:postgres-only native-query-with-temporal-checkpoint-test(testing "Native query with temporal checkpoint";; we test only in postgres because it's easy to cast to ::timestampdiff --git a/test/metabase/driver/postgres_test.clj b/test/metabase/driver/postgres_test.cljindex 188543ba92a..9e5566e583f 100644--- a/test/metabase/driver/postgres_test.clj+++ b/test/metabase/driver/postgres_test.clj@@ -173,7 +173,7 @@;;; ------------------------------------------- Tests for sync edge cases ---------------------------------------------(deftest edge-case-identifiers-test+(deftest ^:sequential edge-case-identifiers-test(mt/test-driver :postgres(testing "Make sure that Tables / Fields with dots in their names get escaped properly"(mt/dataset dots-in-names