The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Add batch_id to data complexity events (#73836)
Chris Truter16573c43558b
diff --git a/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity.clj b/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity.cljindex d87a08600e6..8bac77affbe 100644--- a/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity.clj+++ b/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity.clj@@ -390,6 +390,7 @@Returns false when tracking is disabled or any emission failed."[{:keys [library universe metabot meta]}](let [base {:event :data_complexity_scoring+ :batch_id (str (random-uuid)):formula_version (:formula-version meta):parameters (parameters-map meta)}events (for [[catalog result] [[:library library] [:universe universe] [:metabot metabot]]diff --git a/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.clj b/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.cljindex 079eff6bab5..e16ebc87854 100644--- a/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.clj+++ b/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.clj@@ -793,6 +793,31 @@(is (every? #(= expected-model (get-in % ["parameters" "embedding_model"])) events))(is (every? #(= "names_split" (get-in % ["parameters" "text_variant"])) events)))))))+(deftest ^:sequential emit-snowplow-batch-id-test+ (testing "scoring stamps every event with a UUID batch_id, constant within a pass and fresh between passes"+ (snowplow-test/with-fake-snowplow-collector+ (mt/with-dynamic-fn-redefs [complexity/enumerate-catalogs+ (constantly {:library [(entity :name "orders")]+ :universe [(entity :name "orders")]+ :metabot [(entity :name "orders")]})]+ (snowplow-test/pop-event-data-and-user-id!)+ (letfn [(drain-and-check-pass! []+ (let [events (complexity-events!)+ batch-ids (set (map #(get % "batch_id") events))+ catalogs (frequencies (map #(get % "catalog") events))+ event-keys (set (map (juxt #(get % "catalog") #(get % "key")) events))]+ ;; 1 grand total + 2 group totals + 5 leaves = 8 events per catalog.+ (is (= {"library" 8 "universe" 8 "metabot" 8} catalogs))+ (is (= (count events) (count event-keys)))+ (is (= 1 (count batch-ids)))+ (is (every? parse-uuid batch-ids))+ (first batch-ids)))]+ (complexity/complexity-scores :embedder nil)+ (let [batch-1 (drain-and-check-pass!)]+ (complexity/complexity-scores :embedder nil)+ (let [batch-2 (drain-and-check-pass!)]+ (is (not= batch-1 batch-2)))))))))+(deftest ^:sequential emit-snowplow-failure-is-swallowed-test(testing "emission failure is caught; complexity-scores still returns the score and logs a warning"(mt/with-dynamic-fn-redefs [complexity/enumerate-catalogsdiff --git a/snowplow/iglu-client-embedded/schemas/com.metabase/data_complexity/jsonschema/1-0-0 b/snowplow/iglu-client-embedded/schemas/com.metabase/data_complexity/jsonschema/1-0-0index 9b75fde183f..cb13c6ed820 100644--- a/snowplow/iglu-client-embedded/schemas/com.metabase/data_complexity/jsonschema/1-0-0+++ b/snowplow/iglu-client-embedded/schemas/com.metabase/data_complexity/jsonschema/1-0-0@@ -11,6 +11,7 @@"additionalProperties": false,"required": ["event",+ "batch_id","catalog","key","formula_version",@@ -25,6 +26,12 @@],"maxLength": 64},+ "batch_id": {+ "description": "Unique identifier for the scoring run that emitted this event; consumers use it to group events from the same run.",+ "type": "string",+ "format": "uuid",+ "maxLength": 36+ },"catalog": {"description": "Which catalog of the data layer was scored.","type": "string",