Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

GDGT-2076 add observability to incremental transforms (#73364)

YB74a33038bac5
diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn
index 36635a5c413..ecad9d49b72 100644
--- a/.clj-kondo/config.edn
+++ b/.clj-kondo/config.edn
@@ -662,6 +662,7 @@
metabase.test/with-temp hooks.toucan2.tools.with-temp/with-temp
metabase.test/with-temp-file hooks.metabase.test.util/with-temp-file
metabase.test/with-temporary-setting-values hooks.metabase.test.util/with-temporary-setting-values
+ metabase.tracing.test-util/with-span-exporter hooks.common/with-one-binding
metabase.users-rest.api-test/with-temp-user-email! hooks.common/with-one-binding
metabase.util.i18n/deferred-trs hooks.metabase.util.i18n/deferred-trs
metabase.util.i18n/deferred-tru hooks.metabase.util.i18n/deferred-tru
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index c9115f3bc3b..779a50871e2 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -2246,7 +2246,8 @@
{:team "DevEx"
:api #{metabase.tracing.core
metabase.tracing.init}
- :uses #{events
+ :uses #{config
+ events
settings
task
util}}
@@ -3445,6 +3446,7 @@
permissions
query-processor
settings
+ tracing
transforms
transforms-base
util}
diff --git a/deps.edn b/deps.edn
index b8a0dd1a48a..f625e64aa75 100644
--- a/deps.edn
+++ b/deps.edn
@@ -266,6 +266,7 @@
integrant/integrant {:mvn/version "1.0.1"} ; component lifecycle management
io.github.camsaul/humane-are {:mvn/version "1.0.2"} ; cleaner test output
io.github.metabase/hawk {:mvn/version "1.0.13"} ; test runner
+ io.opentelemetry/opentelemetry-sdk-testing {:mvn/version "1.54.1"} ; InMemorySpanExporter for tracing tests; pinned to match the resolved OTel SDK version
io.zonky.test/embedded-postgres {:mvn/version "2.2.2"} ; In-process postgres for dev and testing
;; Native postgres binaries for every OS/arch we're likely to run on in
;; dev or CI. Without these Zonky falls back to x86_64 under Rosetta on
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj
index 1620e9be98c..2d7391a1f4d 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj
@@ -170,7 +170,7 @@
metadata temp-file]
;; First incremental run: no checkpoint exists yet, behave like non-incremental
;; to drop and recreate the table rather than appending to existing data.
- (if (nil? (:last_checkpoint_value transform))
+ (if (transforms-base.u/first-incremental-run? transform)
((get-method transfer-file-to-db :table) driver db transform metadata temp-file)
;; Normal incremental: append if table exists, create if it doesn't
(let [table-name (transforms-base.u/qualified-table-name driver target)
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
index 3a16b0c3dda..f1e00b33ec3 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
@@ -7,6 +7,7 @@
[metabase.app-db.core :as app-db]
[metabase.driver :as driver]
[metabase.driver.connection :as driver.conn]
+ [metabase.tracing.core :as tracing]
[metabase.transforms-base.interface :as transforms-base.i]
[metabase.transforms-base.util :as transforms-base.u]
[metabase.transforms.instrumentation :as transforms.instrumentation]
@@ -108,29 +109,37 @@
(some-> start-promise (deliver [:started run-id]))
(with-open [_ (open-python-message-update-future! run-id message-log)]
(driver.conn/with-write-connection
- (let [conn-spec (driver/connection-spec driver db)
- transform-details {:db-id (:id db) :conn-spec conn-spec :output-schema (:schema target)}
- run-fn (fn [cancel-chan source-range-params]
- (let [result (transforms-base.i/execute-base!
- transform
- {:cancelled? #(boolean (a/poll! cancel-chan))
- :cancel-chan cancel-chan
- :run-id run-id
- :source-range-params source-range-params
- :message-log message-log
- :with-stage-timing-fn (fn [rid stage thunk]
- (transforms.instrumentation/with-stage-timing [rid stage]
- (thunk)))})]
- (save-log-to-transform-run-message! run-id message-log)
- (when-not (= :succeeded (:status result))
- (throw (or (:error result) (ex-info "Transform failed" {:status (:status result)}))))
- result))
- ex-message-fn #(exceptional-run-message message-log %)
- result (transforms.instrumentation/with-stage-timing [run-id [:computation :python-execution]]
- (transforms.u/run-cancelable-transform! run-id transform driver transform-details run-fn :ex-message-fn ex-message-fn))]
- ;; Post-processing: sync, transform_id, events
- (transforms-base.u/complete-execution! transform {})
- {:run_id run-id :result result}))))
+ (let [target-type (keyword (:type target))]
+ (tracing/with-span :tasks "task.transform.python"
+ {:transform/id transform-id
+ :transform/target-type (name target-type)
+ :transform/incremental (= :table-incremental target-type)
+ :transform/first-incremental-run (transforms-base.u/first-incremental-run? transform)
+ :db/id (:id db)
+ :db/engine (name driver)}
+ (let [conn-spec (driver/connection-spec driver db)
+ transform-details {:db-id (:id db) :conn-spec conn-spec :output-schema (:schema target)}
+ run-fn (fn [cancel-chan source-range-params]
+ (let [result (transforms-base.i/execute-base!
+ transform
+ {:cancelled? #(boolean (a/poll! cancel-chan))
+ :cancel-chan cancel-chan
+ :run-id run-id
+ :source-range-params source-range-params
+ :message-log message-log
+ :with-stage-timing-fn (fn [rid stage thunk]
+ (transforms.instrumentation/with-stage-timing [rid stage]
+ (thunk)))})]
+ (save-log-to-transform-run-message! run-id message-log)
+ (when-not (= :succeeded (:status result))
+ (throw (or (:error result) (ex-info "Transform failed" {:status (:status result)}))))
+ result))
+ ex-message-fn #(exceptional-run-message message-log %)
+ result (transforms.instrumentation/with-stage-timing [run-id [:computation :python-execution]]
+ (transforms.u/run-cancelable-transform! run-id transform driver transform-details run-fn :ex-message-fn ex-message-fn))]
+ ;; Post-processing: sync, transform_id, events
+ (transforms-base.u/complete-execution! transform {})
+ {:run_id run-id :result result}))))))
(catch Throwable t
(log/error t "Error executing Python transform")
(throw t))))
diff --git a/src/metabase/tracing/core.clj b/src/metabase/tracing/core.clj
index c52ee68e387..e02911bb3d9 100644
--- a/src/metabase/tracing/core.clj
+++ b/src/metabase/tracing/core.clj
@@ -18,6 +18,7 @@
MB_TRACING_GROUPS=qp,api MB_TRACING_SERVICE_NAME=metabase-prod-1"
(:require
[clojure.string :as str]
+ [metabase.config.core :as config]
[metabase.tracing.attributes :as trace-attrs]
[metabase.tracing.settings :as tracing.settings]
[metabase.util.log :as log]
@@ -243,6 +244,37 @@
;;; ------------------------------------------------ Primary Macro -------------------------------------------------
+(def ^:private ^:dynamic *span-attrs*
+ "Atom of attrs already written to the active `with-span`. Used by [[add-span-attrs!]] to detect duplicate writes; nil outside a `with-span`."
+ nil)
+
+(defn- merge-attrs-and-detect-duplicates!
+ "Merge `attrs` into `*span-attrs*` and return the subset that should be written to the active span. Any key already in the atom is a duplicate: throws in dev/test, drops with a `log/warn` in prod. Returns `attrs` unchanged when `*span-attrs*` is unbound (outside `with-span`)."
+ [attrs]
+ (if-let [span-attrs *span-attrs*]
+ (let [snapshot @span-attrs
+ duplicates (filterv #(contains? snapshot %) (keys attrs))
+ new-attrs (apply dissoc attrs duplicates)]
+ (when (seq duplicates)
+ (if (or config/is-dev? config/is-test?)
+ (throw (ex-info (format "Span attribute already set: %s" (pr-str duplicates))
+ {:duplicates duplicates
+ :existing (select-keys snapshot duplicates)
+ :new (select-keys attrs duplicates)}))
+ (log/warnf "Span attribute already set, dropping duplicate writes: %s"
+ (pr-str duplicates))))
+ (swap! span-attrs merge new-attrs)
+ new-attrs)
+ attrs))
+
+(defn add-span-attrs!
+ "Enrich the active OTel span with `attrs` mid-execution. No-op if `group` is disabled or `attrs` is empty. Re-writing a key already on the span is a duplicate: throws in dev/test, dropped (with `log/warn`) in prod."
+ [group attrs]
+ (when (and (group-enabled? group) (seq attrs))
+ (let [new-attrs (merge-attrs-and-detect-duplicates! attrs)]
+ (when (seq new-attrs)
+ (span/add-span-data! {:attributes new-attrs})))))
+
(defmacro with-span
"Create an OTel span if tracing is enabled for `group`.
@@ -263,27 +295,29 @@
(process-query query))"
[group span-name attrs & body]
`(if (group-enabled? ~group)
- (let [prev-trace-id# (ThreadContext/get "trace_id")
+ (let [attrs# ~attrs
+ prev-trace-id# (ThreadContext/get "trace_id")
prev-span-id# (ThreadContext/get "span_id")
root-span?# (nil? prev-span-id#)]
- (span/with-span! {:name ~span-name :attributes ~attrs}
- (inject-trace-id-into-mdc!)
- ;; For root spans (no parent in MDC), set Pyroscope profiling context.
- ;; Nested child spans skip this — the root's context covers all samples.
- (when root-span?#
- (let [^Span span# (Span/current)
- ^SpanContext ctx# (.getSpanContext span#)
- span-id# (.getSpanId ctx#)]
- (set-pyroscope-context! span# span-id# ~span-name)))
- (try
- ~@body
- (finally
- (when root-span?#
- (clear-pyroscope-context!))
- (if prev-trace-id#
- (do (ThreadContext/put "trace_id" prev-trace-id#)
- (ThreadContext/put "span_id" prev-span-id#))
- (clear-trace-id-from-mdc!))))))
+ (binding [*span-attrs* (atom attrs#)]
+ (span/with-span! {:name ~span-name :attributes attrs#}
+ (inject-trace-id-into-mdc!)
+ ;; For root spans (no parent in MDC), set Pyroscope profiling context.
+ ;; Nested child spans skip this — the root's context covers all samples.
+ (when root-span?#
+ (let [^Span span# (Span/current)
+ ^SpanContext ctx# (.getSpanContext span#)
+ span-id# (.getSpanId ctx#)]
+ (set-pyroscope-context! span# span-id# ~span-name)))
+ (try
+ ~@body
+ (finally
+ (when root-span?#
+ (clear-pyroscope-context!))
+ (if prev-trace-id#
+ (do (ThreadContext/put "trace_id" prev-trace-id#)
+ (ThreadContext/put "span_id" prev-span-id#))
+ (clear-trace-id-from-mdc!)))))))
(do ~@body)))
;;; -------------------------------------------- SDK Lifecycle -------------------------------------------------
diff --git a/src/metabase/transforms/query_impl.clj b/src/metabase/transforms/query_impl.clj
index d3f14f705ae..4b128dcff29 100644
--- a/src/metabase/transforms/query_impl.clj
+++ b/src/metabase/transforms/query_impl.clj
@@ -30,31 +30,35 @@
(driver.conn/with-write-connection
(log/info "Executing transform" id "with target" (pr-str target)
(when (driver.conn/write-connection-requested?) " using write connection"))
- (tracing/with-span :tasks "task.transform.query" {:transform/id id
- :transform/target-type (name (keyword (:type target)))
- :db/id (:id db)
- :db/engine (name driver)}
- (let [conn-spec (driver/connection-spec driver db)
- transform-details {:db-id (:id db) :conn-spec conn-spec :output-schema (:schema target)}
- _exec-result
- (transforms.instrumentation/with-stage-timing [run-id [:computation :mbql-query]]
- (transforms.u/run-cancelable-transform!
- run-id transform driver transform-details
- (fn [cancel-chan source-range-params]
- (let [result (transforms-base.i/execute-base!
- transform
- {:cancelled? #(boolean (a/poll! cancel-chan))
- :run-id run-id
- :source-range-params source-range-params
- :with-stage-timing-fn (fn [rid stage thunk]
- (transforms.instrumentation/with-stage-timing [rid stage]
- (thunk)))})]
- ;; Bridge result-map to exception-based flow for run-cancelable-transform!
- (when-not (= :succeeded (:status result))
- (throw (or (:error result) (ex-info "Transform failed" {:status (:status result)}))))
- result))))]
- ;; Post-processing: sync, transform_id, events
- (transforms-base.u/complete-execution! transform {})))))
+ (let [target-type (keyword (:type target))]
+ (tracing/with-span :tasks "task.transform.query"
+ {:transform/id id
+ :transform/target-type (name target-type)
+ :transform/incremental (= :table-incremental target-type)
+ :transform/first-incremental-run (transforms-base.u/first-incremental-run? transform)
+ :db/id (:id db)
+ :db/engine (name driver)}
+ (let [conn-spec (driver/connection-spec driver db)
+ transform-details {:db-id (:id db) :conn-spec conn-spec :output-schema (:schema target)}
+ _exec-result
+ (transforms.instrumentation/with-stage-timing [run-id [:computation :mbql-query]]
+ (transforms.u/run-cancelable-transform!
+ run-id transform driver transform-details
+ (fn [cancel-chan source-range-params]
+ (let [result (transforms-base.i/execute-base!
+ transform
+ {:cancelled? #(boolean (a/poll! cancel-chan))
+ :run-id run-id
+ :source-range-params source-range-params
+ :with-stage-timing-fn (fn [rid stage thunk]
+ (transforms.instrumentation/with-stage-timing [rid stage]
+ (thunk)))})]
+ ;; Bridge result-map to exception-based flow for run-cancelable-transform!
+ (when-not (= :succeeded (:status result))
+ (throw (or (:error result) (ex-info "Transform failed" {:status (:status result)}))))
+ result))))]
+ ;; Post-processing: sync, transform_id, events
+ (transforms-base.u/complete-execution! transform {}))))))
(catch Throwable t
(if (= :already-running (:error (ex-data t)))
(log/warnf "Transform %d is already running" id)
diff --git a/src/metabase/transforms/util.clj b/src/metabase/transforms/util.clj
index f6db0392530..c11236ec722 100644
--- a/src/metabase/transforms/util.clj
+++ b/src/metabase/transforms/util.clj
@@ -13,6 +13,7 @@
[metabase.driver.util :as driver.u]
[metabase.models.interface :as mi]
[metabase.query-processor.pipeline :as qp.pipeline]
+ [metabase.tracing.core :as tracing]
[metabase.transforms-base.util :as transforms-base.u]
[metabase.transforms.canceling :as canceling]
[metabase.transforms.feature-gating :as transforms.gating]
@@ -118,6 +119,10 @@
(driver/create-schema-if-needed! driver conn-spec output-schema))
(let [source-range-params (transforms-base.u/get-source-range-params transform)]
(transforms-base.u/save-run-checkpoint-range! run-id source-range-params)
+ ;; Enrich the active task.transform.* span with checkpoint range attrs for
+ ;; incremental runs. No-op when tracing is disabled or no span is active.
+ (when source-range-params
+ (tracing/add-span-attrs! :tasks (transforms-base.u/checkpoint-span-attrs source-range-params)))
(canceling/chan-start-timeout-vthread! run-id (transforms.settings/transform-timeout))
(let [cancel-chan (a/promise-chan)
ret (binding [qp.pipeline/*canceled-chan* cancel-chan]
diff --git a/src/metabase/transforms_base/query.clj b/src/metabase/transforms_base/query.clj
index aa8ab55e614..44f09040bf1 100644
--- a/src/metabase/transforms_base/query.clj
+++ b/src/metabase/transforms_base/query.clj
@@ -90,8 +90,7 @@
_ (transforms-base.u/throw-if-db-routing-enabled! transform database)
;; First incremental run (no checkpoint) should behave like non-incremental
;; to drop and recreate the table rather than appending to existing data.
- effective-transform-type (if (and (= :table-incremental (keyword (:type target)))
- (nil? (:last_checkpoint_value transform)))
+ effective-transform-type (if (transforms-base.u/first-incremental-run? transform)
:table
(keyword (:type target)))
transform-details {:db-id db
diff --git a/src/metabase/transforms_base/util.clj b/src/metabase/transforms_base/util.clj
index 6590415d4e6..631e951099e 100644
--- a/src/metabase/transforms_base/util.clj
+++ b/src/metabase/transforms_base/util.clj
@@ -162,17 +162,42 @@
(or (isa? base-type :type/Temporal)
(isa? base-type :type/Number)))
+(defn first-incremental-run?
+ "True when `transform` is configured as `:table-incremental` but has not yet recorded a watermark.
+ First incremental runs behave like full runs — they drop-and-recreate the target table rather than
+ appending — so call sites that branch on incremental semantics need to special-case them.
+
+ Note: this stays true across failed first attempts until one successfully persists a watermark,
+ since the predicate only inspects `:last_checkpoint_value`."
+ [{:keys [target last_checkpoint_value]}]
+ (and (= :table-incremental (keyword (:type target)))
+ (nil? last_checkpoint_value)))
+
(defn- encode-checkpoint-value [v]
(if (number? v)
(str v)
(u.date/format v)))
+(defn checkpoint-span-attrs
+ "Build a map of OTel span attributes from `source-range-params`. Returns an empty
+ map when params are nil. Values are encoded as strings (the same encoding used
+ for persistence) so spans render consistently regardless of base type."
+ [source-range-params]
+ (let [{:keys [checkpoint-filter-field-id]
+ {lo-value :value} :lo
+ {hi-value :value} :hi} source-range-params]
+ (cond-> {}
+ checkpoint-filter-field-id (assoc :transform/checkpoint-field-id checkpoint-filter-field-id)
+ (some? lo-value) (assoc :transform/checkpoint-lo (encode-checkpoint-value lo-value))
+ (some? hi-value) (assoc :transform/checkpoint-hi (encode-checkpoint-value hi-value)))))
+
(defn save-watermark!
"Commits the incremental transforms :hi watermark value to the appdb."
[transform-id source-range-params]
- (t2/update! :model/Transform
- transform-id
- {:last_checkpoint_value (some-> source-range-params :hi :value encode-checkpoint-value)}))
+ (let [hi-value (:value (:hi source-range-params))]
+ (t2/update! :model/Transform
+ transform-id
+ {:last_checkpoint_value (some-> hi-value encode-checkpoint-value)})))
(defn save-run-checkpoint-range!
"Persist the checkpoint range (lo/hi) on a transform run record.
@@ -661,7 +686,7 @@
{:temporal t})))))
(defn utc-timestamp-string
- "Convert the timestamp t to a string encoding the it in the system timezone."
+ "Convert the timestamp `t` to a UTC ISO-8601 string."
[t]
(-> t ->instant str))
diff --git a/test/metabase/tracing/core_test.clj b/test/metabase/tracing/core_test.clj
index 755d5ff9091..cc73619f721 100644
--- a/test/metabase/tracing/core_test.clj
+++ b/test/metabase/tracing/core_test.clj
@@ -1,7 +1,9 @@
(ns metabase.tracing.core-test
(:require
[clojure.test :refer [deftest is testing]]
- [metabase.tracing.core :as tracing])
+ [metabase.config.core :as config]
+ [metabase.tracing.core :as tracing]
+ [metabase.tracing.test-util :as tracing.tu])
(:import
(org.apache.logging.log4j ThreadContext)))
@@ -89,6 +91,101 @@
(reset! side-effect true))
(is @side-effect))))
+(deftest add-span-attrs!-disabled-is-noop-test
+ (testing "add-span-attrs! returns nil when the trace group is disabled"
+ ;; Tracing not initialized, so group-enabled? returns false
+ (tracing/shutdown-groups!)
+ (is (nil? (tracing/add-span-attrs! :tasks {:transform/incremental true})))))
+
+(deftest add-span-attrs!-detects-duplicate-keys-test
+ (testing "duplicate against with-span attrs throws (same value)"
+ (tracing.tu/with-span-exporter [_exporter]
+ (try
+ (tracing/init-enabled-groups! "tasks" "INFO")
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo
+ #"Span attribute already set"
+ (tracing/with-span :tasks "test.span" {:foo/k 1}
+ (tracing/add-span-attrs! :tasks {:foo/k 1}))))
+ (finally
+ (tracing/shutdown-groups!)))))
+
+ (testing "duplicate against with-span attrs throws (different value)"
+ (tracing.tu/with-span-exporter [_exporter]
+ (try
+ (tracing/init-enabled-groups! "tasks" "INFO")
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo
+ #"Span attribute already set"
+ (tracing/with-span :tasks "test.span" {:foo/k 1}
+ (tracing/add-span-attrs! :tasks {:foo/k 2}))))
+ (finally
+ (tracing/shutdown-groups!)))))
+
+ (testing "duplicate across two add-span-attrs! calls throws"
+ (tracing.tu/with-span-exporter [_exporter]
+ (try
+ (tracing/init-enabled-groups! "tasks" "INFO")
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo
+ #"Span attribute already set"
+ (tracing/with-span :tasks "test.span" {}
+ (tracing/add-span-attrs! :tasks {:foo/k "first"})
+ (tracing/add-span-attrs! :tasks {:foo/k "second"}))))
+ (finally
+ (tracing/shutdown-groups!)))))
+
+ (testing "distinct keys across with-span and add-span-attrs! are allowed"
+ (tracing.tu/with-span-exporter [exporter]
+ (try
+ (tracing/init-enabled-groups! "tasks" "INFO")
+ (tracing/with-span :tasks "test.span" {:foo/a 1}
+ (tracing/add-span-attrs! :tasks {:foo/b 2}))
+ (let [attrs (-> exporter tracing.tu/finished-spans first :attrs)]
+ (is (= 1 (get attrs "foo.a")))
+ (is (= 2 (get attrs "foo.b"))))
+ (finally
+ (tracing/shutdown-groups!))))))
+
+(deftest add-span-attrs!-prod-mode-drops-duplicates-test
+ (testing "in non-dev/test mode, duplicate keys are dropped (with warn log) and the rest is written"
+ (tracing.tu/with-span-exporter [exporter]
+ (try
+ (tracing/init-enabled-groups! "tasks" "INFO")
+ (with-redefs [config/is-dev? false
+ config/is-test? false]
+ (tracing/with-span :tasks "test.span" {:foo/seed 1}
+ (tracing/add-span-attrs! :tasks {:foo/seed 999 :foo/added 2})))
+ (let [attrs (-> exporter tracing.tu/finished-spans first :attrs)]
+ (is (= 1 (get attrs "foo.seed")) "duplicate seed kept its original value (the second write was dropped)")
+ (is (= 2 (get attrs "foo.added")) "non-duplicate added was written"))
+ (finally
+ (tracing/shutdown-groups!))))))
+
+(deftest add-span-attrs!-merges-onto-finished-span-test
+ (testing "attrs from with-span and add-span-attrs! both land on the same finished span; empty/nil attrs do not contribute"
+ (tracing.tu/with-span-exporter [exporter]
+ (try
+ (tracing/init-enabled-groups! "tasks" "INFO")
+ (tracing/with-span :tasks "test.span" {:foo/id 42 :foo/seed "yes"}
+ (tracing/add-span-attrs! :tasks {:foo/incremental true})
+ (tracing/add-span-attrs! :tasks {:foo/first-incremental-run false})
+ (tracing/add-span-attrs! :tasks {})
+ (tracing/add-span-attrs! :tasks nil))
+ ;; clj-otel encodes namespaced keywords with `.` separator and `-` as `_`,
+ ;; e.g. :foo/first-incremental-run -> "foo.first_incremental_run".
+ ;; It also auto-adds default attrs (thread.*, code.*) on every span, so we filter to our prefix
+ ;; before counting to detect any spurious contributions.
+ (let [finished (tracing.tu/finished-spans exporter)
+ span (first finished)
+ our-attrs (into {} (filter (fn [[^String k _]] (.startsWith k "foo."))) (:attrs span))]
+ (is (= 1 (count finished)))
+ (is (= "test.span" (:name span)))
+ (is (= 4 (count our-attrs)) "exactly the four contributed attrs, nothing extra under our prefix")
+ (is (= 42 (get our-attrs "foo.id")))
+ (is (= "yes" (get our-attrs "foo.seed")))
+ (is (true? (get our-attrs "foo.incremental")))
+ (is (false? (get our-attrs "foo.first_incremental_run"))))
+ (finally
+ (tracing/shutdown-groups!))))))
+
(deftest inject-trace-level-into-mdc-test
(testing "inject-trace-id-into-mdc! sets trace_level in MDC from cached setting"
(try
diff --git a/test/metabase/tracing/test_util.clj b/test/metabase/tracing/test_util.clj
new file mode 100644
index 00000000000..fcb1af4cc69
--- /dev/null
+++ b/test/metabase/tracing/test_util.clj
@@ -0,0 +1,47 @@
+(ns metabase.tracing.test-util
+ "Test helpers for asserting OpenTelemetry span output. Provides an in-memory
+ span exporter wired into clj-otel's default tracer for the duration of a test,
+ plus a Clojure-friendly accessor for reading captured spans back."
+ (:require
+ [steffan-westcott.clj-otel.api.trace.span :as span])
+ (:import
+ (io.opentelemetry.api.common AttributeKey)
+ (io.opentelemetry.sdk.testing.exporter InMemorySpanExporter)
+ (io.opentelemetry.sdk.trace SdkTracerProvider)
+ (io.opentelemetry.sdk.trace.data SpanData)
+ (io.opentelemetry.sdk.trace.export SimpleSpanProcessor)))
+
+(set! *warn-on-reflection* true)
+
+(defmacro with-span-exporter
+ "Run `body` with clj-otel's default tracer pointed at an in-memory span exporter.
+ `exporter-binding` is bound to the exporter — pass it to [[finished-spans]] to
+ read captured spans as Clojure maps.
+
+ SimpleSpanProcessor exports synchronously on span end, so reads after the body
+ are deterministic. On exit, the default tracer is reset to nil (clj-otel's
+ initial state)."
+ [[exporter-binding] & body]
+ `(let [~exporter-binding (InMemorySpanExporter/create)
+ provider# ^SdkTracerProvider (-> (SdkTracerProvider/builder)
+ (.addSpanProcessor (SimpleSpanProcessor/create ~exporter-binding))
+ (.build))]
+ (span/set-default-tracer! (.get provider# "metabase-test"))
+ (try
+ ~@body
+ (finally
+ (span/set-default-tracer! nil)
+ (.shutdown provider#)))))
+
+(defn finished-spans
+ "Return finished spans from an in-memory span exporter as a vector of maps:
+ - :name span name (string)
+ - :attrs map of OTel attribute name (string) -> value
+ For tests needing more SpanData fields, extend this helper or read the raw
+ SpanData via `(.getFinishedSpanItems exporter)`."
+ [^InMemorySpanExporter exporter]
+ (mapv (fn [^SpanData span]
+ {:name (.getName span)
+ :attrs (into {} (map (fn [[^AttributeKey k v]] [(.getKey k) v]))
+ (.asMap (.getAttributes span)))})
+ (.getFinishedSpanItems exporter)))
diff --git a/test/metabase/transforms_base/util_test.clj b/test/metabase/transforms_base/util_test.clj
index e86c11dbbe7..f442fa3e7df 100644
--- a/test/metabase/transforms_base/util_test.clj
+++ b/test/metabase/transforms_base/util_test.clj
@@ -1,6 +1,7 @@
(ns metabase.transforms-base.util-test
(:require
[clojure.test :refer :all]
+ [java-time.api :as t]
[metabase.config.core :as config]
[metabase.test :as mt]
[metabase.transforms-base.util :as transforms-base.u]))
@@ -24,3 +25,41 @@
(transforms-base.u/throw-if-db-routing-enabled!
{:name "Routing transform"}
(mt/db)))))))))
+
+(deftest ^:parallel first-incremental-run?-test
+ (testing "true for an incremental transform with no checkpoint yet"
+ (is (true? (transforms-base.u/first-incremental-run?
+ {:target {:type "table-incremental"} :last_checkpoint_value nil}))))
+ (testing "false for an incremental transform that has already recorded a watermark"
+ (is (false? (transforms-base.u/first-incremental-run?
+ {:target {:type "table-incremental"} :last_checkpoint_value "42"}))))
+ (testing "false for non-incremental targets regardless of checkpoint value"
+ (is (false? (transforms-base.u/first-incremental-run?
+ {:target {:type "table"} :last_checkpoint_value nil})))
+ (is (false? (transforms-base.u/first-incremental-run?
+ {:target {:type :table} :last_checkpoint_value nil}))))
+ (testing "accepts both string and keyword target types"
+ (is (true? (transforms-base.u/first-incremental-run?
+ {:target {:type :table-incremental} :last_checkpoint_value nil})))))
+
+(deftest ^:parallel checkpoint-span-attrs-test
+ (testing "nil source-range-params yields an empty attrs map"
+ (is (= {} (transforms-base.u/checkpoint-span-attrs nil))))
+ (testing "field-id only (no lo/hi) yields just :transform/checkpoint-field-id"
+ (is (= {:transform/checkpoint-field-id 42}
+ (transforms-base.u/checkpoint-span-attrs {:checkpoint-filter-field-id 42}))))
+ (testing "numeric lo/hi are encoded as strings"
+ (is (= {:transform/checkpoint-field-id 7
+ :transform/checkpoint-lo "10"
+ :transform/checkpoint-hi "100"}
+ (transforms-base.u/checkpoint-span-attrs
+ {:checkpoint-filter-field-id 7
+ :lo {:value 10}
+ :hi {:value 100}}))))
+ (testing "temporal hi is formatted as an ISO string"
+ (let [attrs (transforms-base.u/checkpoint-span-attrs
+ {:checkpoint-filter-field-id 9
+ :hi {:value (t/local-date-time 2024 1 16 10 0 0)}})]
+ (is (= 9 (:transform/checkpoint-field-id attrs)))
+ (is (string? (:transform/checkpoint-hi attrs)))
+ (is (re-find #"2024-01-16" (:transform/checkpoint-hi attrs))))))