Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

transforms: pass full object instead of re-querying db internally (#68456)

Oleksandr Solovyovac421d0a6d1e
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/util.clj b/enterprise/backend/src/metabase_enterprise/transforms/util.clj
index 0891687d0fa..375cf0e8f98 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/util.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/util.clj
@@ -238,9 +238,8 @@
Returns a map with `:query` (MBQL query selecting the max) and `:filter-column` (column metadata),
or `nil` if the transform doesn't use checkpoint-based incremental strategy or the target table doesn't exist."
- [transform-id]
- (let [{:keys [source target] :as transform} (t2/select-one :model/Transform transform-id)
- db-id (transforms.i/target-db-id transform)]
+ [{:keys [source target] :as transform}]
+ (let [db-id (transforms.i/target-db-id transform)]
(when (checkpoint-incremental? source)
(when-let [table (target-table db-id target)]
(let [metadata-provider (lib-be/application-database-metadata-provider db-id)
@@ -313,11 +312,11 @@
(defn compile-source
"Compile the source query of a transform to SQL, applying incremental filtering if required."
- [{:keys [id source]}]
+ [{:keys [source] :as transform}]
(let [{:keys [source-incremental-strategy] query-type :type} source]
(case (keyword query-type)
:query
- (let [checkpoint (next-checkpoint id)
+ (let [checkpoint (next-checkpoint transform)
query (:query source)
driver (some->> query :database (t2/select-one :model/Database) :engine keyword)]
(binding [driver/*compile-with-inline-parameters*
@@ -627,7 +626,7 @@
(when (driver.u/supports? (:engine database) :describe-indexes database)
(let [driver (:engine database)
indexes (driver/describe-table-indexes driver database target)
- checkpoint (next-checkpoint (:id transform))
+ checkpoint (next-checkpoint transform)
{:keys [drop create]}
(decide-secondary-index-ddl
{:filter-column (:filter-column checkpoint)
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj
index d20adcda0dc..5501d52e3e0 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj
@@ -271,9 +271,10 @@
[table-id source-incremental-strategy transform-id limit]
(let [db-id (t2/select-one-fn :db_id (t2/table-name :model/Table) :id table-id)
metadata-provider (lib-be/application-database-metadata-provider db-id)
- table-metadata (lib.metadata/table metadata-provider table-id)]
+ table-metadata (lib.metadata/table metadata-provider table-id)
+ transform (t2/select-one :model/Transform transform-id)]
(cond-> (lib/query metadata-provider table-metadata)
- source-incremental-strategy (transforms.u/preprocess-incremental-query source-incremental-strategy (transforms.u/next-checkpoint transform-id))
+ source-incremental-strategy (transforms.u/preprocess-incremental-query source-incremental-strategy (transforms.u/next-checkpoint transform))
limit (lib/limit limit))))
;; TODO break this up such that s3 can be swapped out for other transfer mechanisms.
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj
index 665ba5e89d8..cb438e720bb 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj
@@ -3,6 +3,7 @@
[clojure.string :as str]
[clojure.test :refer :all]
[metabase-enterprise.transforms-python.execute :as transforms-python.execute]
+ [metabase-enterprise.transforms-python.init]
[metabase-enterprise.transforms-python.python-runner :as transforms-python.python-runner]
[metabase-enterprise.transforms.test-dataset :as transforms-dataset]
[metabase-enterprise.transforms.test-util :as transforms.tu :refer [with-transform-cleanup! get-test-schema]]
@@ -17,6 +18,8 @@
(set! *warn-on-reflection* true)
+(def fast-log-polling-ms "A bit of time to wait for when running periodic processes" 50)
+
(deftest create-python-transform-test
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
(mt/dataset transforms-dataset/transforms-test
@@ -165,27 +168,31 @@
"Polls the `:message` state of the last run and stores the value every time it is different to the last observation.
Close with .close, grab the vector of values with deref."
^Closeable [transform-id]
- (let [states (atom [])
- fut (future
- (try
- (loop []
- (let [{:keys [message]} (get-last-run transform-id)]
- (cond
- (.isInterrupted (Thread/currentThread))
- nil
-
- ;; same message as last time
- (= message (peek @states))
- (recur)
-
- ;; new message value
- :else (do (swap! states conj message)
- (recur)))))
- (catch InterruptedException _ nil)))]
+ (let [states (atom [])
+ stopped? (atom false)
+ fut (future
+ (try
+ (loop []
+ (Thread/sleep ^long fast-log-polling-ms) ; avoid hammering the API
+ (when-not @stopped?
+ (let [{:keys [message]} (get-last-run transform-id)]
+ (cond
+ (.isInterrupted (Thread/currentThread))
+ nil
+
+ ;; same message as last time
+ (= message (peek @states))
+ (recur)
+
+ ;; new message value
+ :else (do (swap! states conj message)
+ (recur))))))
+ (catch InterruptedException _ nil)))]
(reify IDeref
(deref [_] @states)
Closeable
(close [_]
+ (reset! stopped? true)
(future-cancel fut)
(assert (not= :timeout (try (deref fut 1000 :timeout) (catch Throwable _))) "Observation thread did not exit!")))))
@@ -217,7 +224,7 @@
(transforms.tu/wait-for-table (:name target) 5000)))
(run-scenario [scenario schema]
- (with-redefs [transforms-python.execute/python-message-loop-sleep-duration Duration/ZERO
+ (with-redefs [transforms-python.execute/python-message-loop-sleep-duration (Duration/ofMillis fast-log-polling-ms)
transforms-python.execute/transfer-file-to-db (if-some [e (:writeback-ex scenario)]
(fn [& _] (throw e))
@#'transforms-python.execute/transfer-file-to-db)]
@@ -451,7 +458,7 @@
(mt/with-premium-features #{:transforms :transforms-python}
(mt/dataset transforms-dataset/transforms-test
(let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))]
- (with-redefs [transforms-python.execute/python-message-loop-sleep-duration Duration/ZERO]
+ (with-redefs [transforms-python.execute/python-message-loop-sleep-duration (Duration/ofMillis fast-log-polling-ms)]
(with-transform-cleanup! [target {:type "table"
:schema schema
:name "result"}]