Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Add batch_id to data complexity events (#73836)

Chris Truter16573c43558b
diff --git a/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity.clj b/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity.clj
index d87a08600e6..8bac77affbe 100644
--- a/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity.clj
+++ b/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity.clj
@@ -390,6 +390,7 @@
Returns false when tracking is disabled or any emission failed."
[{:keys [library universe metabot meta]}]
(let [base {:event :data_complexity_scoring
+ :batch_id (str (random-uuid))
:formula_version (:formula-version meta)
:parameters (parameters-map meta)}
events (for [[catalog result] [[:library library] [:universe universe] [:metabot metabot]]
diff --git a/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.clj b/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.clj
index 079eff6bab5..e16ebc87854 100644
--- a/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.clj
@@ -793,6 +793,31 @@
(is (every? #(= expected-model (get-in % ["parameters" "embedding_model"])) events))
(is (every? #(= "names_split" (get-in % ["parameters" "text_variant"])) events)))))))
+(deftest ^:sequential emit-snowplow-batch-id-test
+ (testing "scoring stamps every event with a UUID batch_id, constant within a pass and fresh between passes"
+ (snowplow-test/with-fake-snowplow-collector
+ (mt/with-dynamic-fn-redefs [complexity/enumerate-catalogs
+ (constantly {:library [(entity :name "orders")]
+ :universe [(entity :name "orders")]
+ :metabot [(entity :name "orders")]})]
+ (snowplow-test/pop-event-data-and-user-id!)
+ (letfn [(drain-and-check-pass! []
+ (let [events (complexity-events!)
+ batch-ids (set (map #(get % "batch_id") events))
+ catalogs (frequencies (map #(get % "catalog") events))
+ event-keys (set (map (juxt #(get % "catalog") #(get % "key")) events))]
+ ;; 1 grand total + 2 group totals + 5 leaves = 8 events per catalog.
+ (is (= {"library" 8 "universe" 8 "metabot" 8} catalogs))
+ (is (= (count events) (count event-keys)))
+ (is (= 1 (count batch-ids)))
+ (is (every? parse-uuid batch-ids))
+ (first batch-ids)))]
+ (complexity/complexity-scores :embedder nil)
+ (let [batch-1 (drain-and-check-pass!)]
+ (complexity/complexity-scores :embedder nil)
+ (let [batch-2 (drain-and-check-pass!)]
+ (is (not= batch-1 batch-2)))))))))
+
(deftest ^:sequential emit-snowplow-failure-is-swallowed-test
(testing "emission failure is caught; complexity-scores still returns the score and logs a warning"
(mt/with-dynamic-fn-redefs [complexity/enumerate-catalogs
diff --git a/snowplow/iglu-client-embedded/schemas/com.metabase/data_complexity/jsonschema/1-0-0 b/snowplow/iglu-client-embedded/schemas/com.metabase/data_complexity/jsonschema/1-0-0
index 9b75fde183f..cb13c6ed820 100644
--- a/snowplow/iglu-client-embedded/schemas/com.metabase/data_complexity/jsonschema/1-0-0
+++ b/snowplow/iglu-client-embedded/schemas/com.metabase/data_complexity/jsonschema/1-0-0
@@ -11,6 +11,7 @@
"additionalProperties": false,
"required": [
"event",
+ "batch_id",
"catalog",
"key",
"formula_version",
@@ -25,6 +26,12 @@
],
"maxLength": 64
},
+ "batch_id": {
+ "description": "Unique identifier for the scoring run that emitted this event; consumers use it to group events from the same run.",
+ "type": "string",
+ "format": "uuid",
+ "maxLength": 36
+ },
"catalog": {
"description": "Which catalog of the data layer was scored.",
"type": "string",