The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
transforms: pass full object instead of re-querying db internally (#68456)
Oleksandr Solovyovac421d0a6d1e
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/util.clj b/enterprise/backend/src/metabase_enterprise/transforms/util.cljindex 0891687d0fa..375cf0e8f98 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/util.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/util.clj@@ -238,9 +238,8 @@Returns a map with `:query` (MBQL query selecting the max) and `:filter-column` (column metadata),or `nil` if the transform doesn't use checkpoint-based incremental strategy or the target table doesn't exist."- [transform-id]- (let [{:keys [source target] :as transform} (t2/select-one :model/Transform transform-id)- db-id (transforms.i/target-db-id transform)]+ [{:keys [source target] :as transform}]+ (let [db-id (transforms.i/target-db-id transform)](when (checkpoint-incremental? source)(when-let [table (target-table db-id target)](let [metadata-provider (lib-be/application-database-metadata-provider db-id)@@ -313,11 +312,11 @@(defn compile-source"Compile the source query of a transform to SQL, applying incremental filtering if required."- [{:keys [id source]}]+ [{:keys [source] :as transform}](let [{:keys [source-incremental-strategy] query-type :type} source](case (keyword query-type):query- (let [checkpoint (next-checkpoint id)+ (let [checkpoint (next-checkpoint transform)query (:query source)driver (some->> query :database (t2/select-one :model/Database) :engine keyword)](binding [driver/*compile-with-inline-parameters*@@ -627,7 +626,7 @@(when (driver.u/supports? (:engine database) :describe-indexes database)(let [driver (:engine database)indexes (driver/describe-table-indexes driver database target)- checkpoint (next-checkpoint (:id transform))+ checkpoint (next-checkpoint transform){:keys [drop create]}(decide-secondary-index-ddl{:filter-column (:filter-column checkpoint)diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.cljindex d20adcda0dc..5501d52e3e0 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj@@ -271,9 +271,10 @@[table-id source-incremental-strategy transform-id limit](let [db-id (t2/select-one-fn :db_id (t2/table-name :model/Table) :id table-id)metadata-provider (lib-be/application-database-metadata-provider db-id)- table-metadata (lib.metadata/table metadata-provider table-id)]+ table-metadata (lib.metadata/table metadata-provider table-id)+ transform (t2/select-one :model/Transform transform-id)](cond-> (lib/query metadata-provider table-metadata)- source-incremental-strategy (transforms.u/preprocess-incremental-query source-incremental-strategy (transforms.u/next-checkpoint transform-id))+ source-incremental-strategy (transforms.u/preprocess-incremental-query source-incremental-strategy (transforms.u/next-checkpoint transform))limit (lib/limit limit))));; TODO break this up such that s3 can be swapped out for other transfer mechanisms.diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.cljindex 665ba5e89d8..cb438e720bb 100644--- a/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj@@ -3,6 +3,7 @@[clojure.string :as str][clojure.test :refer :all][metabase-enterprise.transforms-python.execute :as transforms-python.execute]+ [metabase-enterprise.transforms-python.init][metabase-enterprise.transforms-python.python-runner :as transforms-python.python-runner][metabase-enterprise.transforms.test-dataset :as transforms-dataset][metabase-enterprise.transforms.test-util :as transforms.tu :refer [with-transform-cleanup! get-test-schema]]@@ -17,6 +18,8 @@(set! *warn-on-reflection* true)+(def fast-log-polling-ms "A bit of time to wait for when running periodic processes" 50)+(deftest create-python-transform-test(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)(mt/dataset transforms-dataset/transforms-test@@ -165,27 +168,31 @@"Polls the `:message` state of the last run and stores the value every time it is different to the last observation.Close with .close, grab the vector of values with deref."^Closeable [transform-id]- (let [states (atom [])- fut (future- (try- (loop []- (let [{:keys [message]} (get-last-run transform-id)]- (cond- (.isInterrupted (Thread/currentThread))- nil-- ;; same message as last time- (= message (peek @states))- (recur)-- ;; new message value- :else (do (swap! states conj message)- (recur)))))- (catch InterruptedException _ nil)))]+ (let [states (atom [])+ stopped? (atom false)+ fut (future+ (try+ (loop []+ (Thread/sleep ^long fast-log-polling-ms) ; avoid hammering the API+ (when-not @stopped?+ (let [{:keys [message]} (get-last-run transform-id)]+ (cond+ (.isInterrupted (Thread/currentThread))+ nil++ ;; same message as last time+ (= message (peek @states))+ (recur)++ ;; new message value+ :else (do (swap! states conj message)+ (recur))))))+ (catch InterruptedException _ nil)))](reify IDeref(deref [_] @states)Closeable(close [_]+ (reset! stopped? true)(future-cancel fut)(assert (not= :timeout (try (deref fut 1000 :timeout) (catch Throwable _))) "Observation thread did not exit!")))))@@ -217,7 +224,7 @@(transforms.tu/wait-for-table (:name target) 5000)))(run-scenario [scenario schema]- (with-redefs [transforms-python.execute/python-message-loop-sleep-duration Duration/ZERO+ (with-redefs [transforms-python.execute/python-message-loop-sleep-duration (Duration/ofMillis fast-log-polling-ms)transforms-python.execute/transfer-file-to-db (if-some [e (:writeback-ex scenario)](fn [& _] (throw e))@#'transforms-python.execute/transfer-file-to-db)]@@ -451,7 +458,7 @@(mt/with-premium-features #{:transforms :transforms-python}(mt/dataset transforms-dataset/transforms-test(let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))]- (with-redefs [transforms-python.execute/python-message-loop-sleep-duration Duration/ZERO]+ (with-redefs [transforms-python.execute/python-message-loop-sleep-duration (Duration/ofMillis fast-log-polling-ms)](with-transform-cleanup! [target {:type "table":schema schema:name "result"}]