The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
GDGT-2017 write watermark to the DB before setting "succeed" status t… (#71334)
Egor Grushina4db64b01b67
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.cljindex f4f680b03d4..3a16b0c3dda 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj@@ -128,13 +128,8 @@ex-message-fn #(exceptional-run-message message-log %)result (transforms.instrumentation/with-stage-timing [run-id [:computation :python-execution]](transforms.u/run-cancelable-transform! run-id transform driver transform-details run-fn :ex-message-fn ex-message-fn))]- ;; Post-processing: sync, transform_id, events, secondary indexes (after succeed-started-run!)- (transforms-base.u/complete-execution! transform- {:run-id run-id- :source-range-params (:source-range-params result)- :with-stage-timing-fn (fn [rid stage thunk]- (transforms.instrumentation/with-stage-timing [rid stage]- (thunk)))})+ ;; Post-processing: sync, transform_id, events+ (transforms-base.u/complete-execution! transform {}){:run_id run-id :result result}))))(catch Throwable t(log/error t "Error executing Python transform")diff --git a/frontend/src/metabase/transforms/hooks/use-transform-with-polling.ts b/frontend/src/metabase/transforms/hooks/use-transform-with-polling.tsindex e035d7601c4..7dac5342e17 100644--- a/frontend/src/metabase/transforms/hooks/use-transform-with-polling.ts+++ b/frontend/src/metabase/transforms/hooks/use-transform-with-polling.ts@@ -1,4 +1,4 @@-import { useEffect, useRef, useState } from "react";+import { useState } from "react";import { skipToken, useGetTransformQuery } from "metabase/api";import type { Transform } from "metabase-types/api";@@ -18,13 +18,11 @@ const isPollingNeeded = (transform?: Transform) =>export const useTransformWithPolling = (transformId: number | undefined) => {const [isPolling, setIsPolling] = useState(false);- const wasPolling = useRef(false);const {data: transform,isLoading,error,- refetch,} = useGetTransformQuery(transformId ?? skipToken, {pollingInterval: isPolling ? POLLING_INTERVAL : undefined,});@@ -33,15 +31,5 @@ export const useTransformWithPolling = (transformId: number | undefined) => {setIsPolling(isPollingNeeded(transform));}- // When polling stops (run just finished), do a delayed refetch to pick up- // post-execution changes like the updated checkpoint watermark.- useEffect(() => {- if (wasPolling.current && !isPolling) {- const timer = setTimeout(() => refetch(), POLLING_INTERVAL);- return () => clearTimeout(timer);- }- wasPolling.current = isPolling;- }, [isPolling, refetch]);-return { transform, isLoading, error };};diff --git a/src/metabase/models/transforms/transform.clj b/src/metabase/models/transforms/transform.cljindex 704294dd47e..4cc36f24454 100644--- a/src/metabase/models/transforms/transform.clj+++ b/src/metabase/models/transforms/transform.clj@@ -186,7 +186,17 @@transforms(let [transform-ids (into #{} (map :id) transforms)last-runs (m/index-by :transform_id (transform-run/latest-runs transform-ids))]- (for [transform transforms] (assoc transform :last_run (get last-runs (:id transform)))))))+ (for [{transform-id :id :as transform} transforms]+ (let [{:keys [status checkpoint_hi_value] :as last-run} (get last-runs transform-id)+ transform (assoc transform :last_run last-run)]+ (if (and (= status :succeeded) checkpoint_hi_value)+ ;; ensure consistency of last_checkpoint_value with last_run+ (if (:last_checkpoint_value transform)+ (assoc transform :last_checkpoint_value checkpoint_hi_value)+ ;; latest transform value wins, could be reset+ (assoc transform :last_checkpoint_value+ (t2/select-one-fn :last_checkpoint_value [:model/Transform :last_checkpoint_value] transform-id)))+ transform))))))(methodical/defmethod t2/batched-hydrate [:model/Transform :transform_tag_ids]"Add tag_ids to a transform, preserving the order defined by position"diff --git a/src/metabase/transforms/query_impl.clj b/src/metabase/transforms/query_impl.cljindex bce1e9f32be..d3f14f705ae 100644--- a/src/metabase/transforms/query_impl.clj+++ b/src/metabase/transforms/query_impl.clj@@ -36,7 +36,7 @@:db/engine (name driver)}(let [conn-spec (driver/connection-spec driver db)transform-details {:db-id (:id db) :conn-spec conn-spec :output-schema (:schema target)}- exec-result+ _exec-result(transforms.instrumentation/with-stage-timing [run-id [:computation :mbql-query]](transforms.u/run-cancelable-transform!run-id transform driver transform-details@@ -53,13 +53,8 @@(when-not (= :succeeded (:status result))(throw (or (:error result) (ex-info "Transform failed" {:status (:status result)}))))result))))]- ;; Post-processing: sync, transform_id, events, secondary indexes (after succeed-started-run!)- (transforms-base.u/complete-execution! transform- {:run-id run-id- :source-range-params (:source-range-params exec-result)- :with-stage-timing-fn (fn [rid stage thunk]- (transforms.instrumentation/with-stage-timing [rid stage]- (thunk)))})))))+ ;; Post-processing: sync, transform_id, events+ (transforms-base.u/complete-execution! transform {})))))(catch Throwable t(if (= :already-running (:error (ex-data t)))(log/warnf "Transform %d is already running" id)diff --git a/src/metabase/transforms/util.clj b/src/metabase/transforms/util.cljindex e302b750786..3642e83cacc 100644--- a/src/metabase/transforms/util.clj+++ b/src/metabase/transforms/util.clj@@ -123,6 +123,7 @@ret (binding [qp.pipeline/*canceled-chan* cancel-chan](canceling/chan-start-run! run-id cancel-chan)(run-transform! cancel-chan source-range-params))]+ (transforms-base.u/save-watermark! (:id transform) source-range-params)(transform-run/succeed-started-run! run-id)ret))(catch Throwable tdiff --git a/src/metabase/transforms_base/core.clj b/src/metabase/transforms_base/core.cljindex 84926b47f17..df3f8e654d7 100644--- a/src/metabase/transforms_base/core.clj+++ b/src/metabase/transforms_base/core.clj@@ -42,6 +42,5 @@([transform opts](let [result (transforms-base.i/execute-base! transform opts)](when (= :succeeded (:status result))- (transforms-base.u/complete-execution! transform- (assoc opts :source-range-params (:source-range-params result))))+ (transforms-base.u/complete-execution! transform opts))result)))diff --git a/src/metabase/transforms_base/util.clj b/src/metabase/transforms_base/util.cljindex a218d36b779..a128b8e5f67 100644--- a/src/metabase/transforms_base/util.clj+++ b/src/metabase/transforms_base/util.clj@@ -459,7 +459,6 @@"Post-processing steps after a transform has been executed successfully.Performs:- - Save watermark (if source-range-params provided)- Sync target table to AppDB- Set `transform_id` on the target table- Publish Metabase events (unless `:publish-events?` is false)@@ -470,13 +469,10 @@to preserve the correct order of operations."[transform opts](let [{:keys [target]} transform- {:keys [publish-events? source-range-params]+ {:keys [publish-events?]:or {publish-events? true}} optsdb-id (transforms-base.i/target-db-id transform)database (t2/select-one :model/Database db-id)]- ;; Save watermark if source-range-params available- (when source-range-params- (save-watermark! (:id transform) source-range-params));; Sync target table(sync-target! target database);; Mark the table as owned by this transform