Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

GDGT-2017 write watermark to the DB before setting "succeed" status t… (#71334)

Egor Grushina4db64b01b67
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
index f4f680b03d4..3a16b0c3dda 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
@@ -128,13 +128,8 @@
ex-message-fn #(exceptional-run-message message-log %)
result (transforms.instrumentation/with-stage-timing [run-id [:computation :python-execution]]
(transforms.u/run-cancelable-transform! run-id transform driver transform-details run-fn :ex-message-fn ex-message-fn))]
- ;; Post-processing: sync, transform_id, events, secondary indexes (after succeed-started-run!)
- (transforms-base.u/complete-execution! transform
- {:run-id run-id
- :source-range-params (:source-range-params result)
- :with-stage-timing-fn (fn [rid stage thunk]
- (transforms.instrumentation/with-stage-timing [rid stage]
- (thunk)))})
+ ;; Post-processing: sync, transform_id, events
+ (transforms-base.u/complete-execution! transform {})
{:run_id run-id :result result}))))
(catch Throwable t
(log/error t "Error executing Python transform")
diff --git a/frontend/src/metabase/transforms/hooks/use-transform-with-polling.ts b/frontend/src/metabase/transforms/hooks/use-transform-with-polling.ts
index e035d7601c4..7dac5342e17 100644
--- a/frontend/src/metabase/transforms/hooks/use-transform-with-polling.ts
+++ b/frontend/src/metabase/transforms/hooks/use-transform-with-polling.ts
@@ -1,4 +1,4 @@
-import { useEffect, useRef, useState } from "react";
+import { useState } from "react";
import { skipToken, useGetTransformQuery } from "metabase/api";
import type { Transform } from "metabase-types/api";
@@ -18,13 +18,11 @@ const isPollingNeeded = (transform?: Transform) =>
export const useTransformWithPolling = (transformId: number | undefined) => {
const [isPolling, setIsPolling] = useState(false);
- const wasPolling = useRef(false);
const {
data: transform,
isLoading,
error,
- refetch,
} = useGetTransformQuery(transformId ?? skipToken, {
pollingInterval: isPolling ? POLLING_INTERVAL : undefined,
});
@@ -33,15 +31,5 @@ export const useTransformWithPolling = (transformId: number | undefined) => {
setIsPolling(isPollingNeeded(transform));
}
- // When polling stops (run just finished), do a delayed refetch to pick up
- // post-execution changes like the updated checkpoint watermark.
- useEffect(() => {
- if (wasPolling.current && !isPolling) {
- const timer = setTimeout(() => refetch(), POLLING_INTERVAL);
- return () => clearTimeout(timer);
- }
- wasPolling.current = isPolling;
- }, [isPolling, refetch]);
-
return { transform, isLoading, error };
};
diff --git a/src/metabase/models/transforms/transform.clj b/src/metabase/models/transforms/transform.clj
index 704294dd47e..4cc36f24454 100644
--- a/src/metabase/models/transforms/transform.clj
+++ b/src/metabase/models/transforms/transform.clj
@@ -186,7 +186,17 @@
transforms
(let [transform-ids (into #{} (map :id) transforms)
last-runs (m/index-by :transform_id (transform-run/latest-runs transform-ids))]
- (for [transform transforms] (assoc transform :last_run (get last-runs (:id transform)))))))
+ (for [{transform-id :id :as transform} transforms]
+ (let [{:keys [status checkpoint_hi_value] :as last-run} (get last-runs transform-id)
+ transform (assoc transform :last_run last-run)]
+ (if (and (= status :succeeded) checkpoint_hi_value)
+ ;; ensure consistency of last_checkpoint_value with last_run
+ (if (:last_checkpoint_value transform)
+ (assoc transform :last_checkpoint_value checkpoint_hi_value)
+ ;; latest transform value wins, could be reset
+ (assoc transform :last_checkpoint_value
+ (t2/select-one-fn :last_checkpoint_value [:model/Transform :last_checkpoint_value] transform-id)))
+ transform))))))
(methodical/defmethod t2/batched-hydrate [:model/Transform :transform_tag_ids]
"Add tag_ids to a transform, preserving the order defined by position"
diff --git a/src/metabase/transforms/query_impl.clj b/src/metabase/transforms/query_impl.clj
index bce1e9f32be..d3f14f705ae 100644
--- a/src/metabase/transforms/query_impl.clj
+++ b/src/metabase/transforms/query_impl.clj
@@ -36,7 +36,7 @@
:db/engine (name driver)}
(let [conn-spec (driver/connection-spec driver db)
transform-details {:db-id (:id db) :conn-spec conn-spec :output-schema (:schema target)}
- exec-result
+ _exec-result
(transforms.instrumentation/with-stage-timing [run-id [:computation :mbql-query]]
(transforms.u/run-cancelable-transform!
run-id transform driver transform-details
@@ -53,13 +53,8 @@
(when-not (= :succeeded (:status result))
(throw (or (:error result) (ex-info "Transform failed" {:status (:status result)}))))
result))))]
- ;; Post-processing: sync, transform_id, events, secondary indexes (after succeed-started-run!)
- (transforms-base.u/complete-execution! transform
- {:run-id run-id
- :source-range-params (:source-range-params exec-result)
- :with-stage-timing-fn (fn [rid stage thunk]
- (transforms.instrumentation/with-stage-timing [rid stage]
- (thunk)))})))))
+ ;; Post-processing: sync, transform_id, events
+ (transforms-base.u/complete-execution! transform {})))))
(catch Throwable t
(if (= :already-running (:error (ex-data t)))
(log/warnf "Transform %d is already running" id)
diff --git a/src/metabase/transforms/util.clj b/src/metabase/transforms/util.clj
index e302b750786..3642e83cacc 100644
--- a/src/metabase/transforms/util.clj
+++ b/src/metabase/transforms/util.clj
@@ -123,6 +123,7 @@
ret (binding [qp.pipeline/*canceled-chan* cancel-chan]
(canceling/chan-start-run! run-id cancel-chan)
(run-transform! cancel-chan source-range-params))]
+ (transforms-base.u/save-watermark! (:id transform) source-range-params)
(transform-run/succeed-started-run! run-id)
ret))
(catch Throwable t
diff --git a/src/metabase/transforms_base/core.clj b/src/metabase/transforms_base/core.clj
index 84926b47f17..df3f8e654d7 100644
--- a/src/metabase/transforms_base/core.clj
+++ b/src/metabase/transforms_base/core.clj
@@ -42,6 +42,5 @@
([transform opts]
(let [result (transforms-base.i/execute-base! transform opts)]
(when (= :succeeded (:status result))
- (transforms-base.u/complete-execution! transform
- (assoc opts :source-range-params (:source-range-params result))))
+ (transforms-base.u/complete-execution! transform opts))
result)))
diff --git a/src/metabase/transforms_base/util.clj b/src/metabase/transforms_base/util.clj
index a218d36b779..a128b8e5f67 100644
--- a/src/metabase/transforms_base/util.clj
+++ b/src/metabase/transforms_base/util.clj
@@ -459,7 +459,6 @@
"Post-processing steps after a transform has been executed successfully.
Performs:
- - Save watermark (if source-range-params provided)
- Sync target table to AppDB
- Set `transform_id` on the target table
- Publish Metabase events (unless `:publish-events?` is false)
@@ -470,13 +469,10 @@
to preserve the correct order of operations."
[transform opts]
(let [{:keys [target]} transform
- {:keys [publish-events? source-range-params]
+ {:keys [publish-events?]
:or {publish-events? true}} opts
db-id (transforms-base.i/target-db-id transform)
database (t2/select-one :model/Database db-id)]
- ;; Save watermark if source-range-params available
- (when source-range-params
- (save-watermark! (:id transform) source-range-params))
;; Sync target table
(sync-target! target database)
;; Mark the table as owned by this transform