Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Add Git-synced content counts to the anonymous stats ping (#84572)

dpsutton65afc667cf07
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj
index b036956cac0..ed30c2c1298 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj
@@ -297,6 +297,11 @@
[card-ids :- [:sequential ::lib.schema.id/card]]
(t2/select [:model/Card :id :type :display] :id [:in card-ids]))
+(mu/defn card-counts-by-type :- [:map-of :keyword :int]
+ "The number of Cards with `card-ids`, by `:type`."
+ [card-ids :- [:sequential ::lib.schema.id/card]]
+ (t2/select-fn->fn :type :count [:model/Card :type [:%count.* :count]] :id [:in card-ids] {:group-by [:type]}))
+
(mu/defn published-table-ids :- [:set ::lib.schema.id/table]
"The ids of the Tables published in the Collections with `collection-ids`."
[collection-ids :- [:sequential ::lib.schema.id/collection]]
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/stats.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/stats.clj
new file mode 100644
index 00000000000..5b4b8f9e0e0
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/stats.clj
@@ -0,0 +1,47 @@
+(ns metabase-enterprise.remote-sync.stats
+ "Counts of Git-synced content for the anonymous stats ping."
+ (:require
+ [metabase-enterprise.remote-sync.db :as remote-sync.db]
+ [metabase-enterprise.remote-sync.impl :as impl]
+ [metabase-enterprise.remote-sync.settings :as settings]
+ [metabase-enterprise.remote-sync.spec :as spec]
+ [metabase.premium-features.core :refer [defenterprise]]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private model-count-keys
+ {"Action" :remote-sync-action-count
+ "Dashboard" :remote-sync-dashboard-count
+ "DataApp" :remote-sync-data-app-count
+ "Dimension" :remote-sync-dimension-count
+ "Document" :remote-sync-document-count
+ "FieldUserSettings" :remote-sync-field-user-settings-count
+ "Glossary" :remote-sync-glossary-entry-count
+ "Measure" :remote-sync-measure-count
+ "NativeQuerySnippet" :remote-sync-snippet-count
+ "Segment" :remote-sync-segment-count
+ "TableUserSettings" :remote-sync-table-user-settings-count
+ "Timeline" :remote-sync-timeline-count
+ "Transform" :remote-sync-transform-count})
+
+(defn- card-type-counts
+ [card-ids]
+ (let [counts (transduce (map #(remote-sync.db/card-counts-by-type (vec %)))
+ (partial merge-with +)
+ {}
+ (partition-all impl/app-db-batch-size card-ids))]
+ {:remote-sync-card-count (reduce + 0 (vals counts))
+ :remote-sync-model-count (get counts :model 0)
+ :remote-sync-question-count (get counts :question 0)
+ :remote-sync-metric-count (get counts :metric 0)}))
+
+(defenterprise remote-sync-stats
+ "Counts of the content a full Git sync export serializes, by model and in total, empty without Git sync."
+ :feature :remote-sync
+ []
+ (if (settings/remote-sync-enabled)
+ (let [entities (spec/exportable-entities)]
+ (merge (into {} (map (fn [[model k]] [k (count (get entities model))])) model-count-keys)
+ (card-type-counts (get entities "Card"))
+ {:remote-sync-content-count (spec/exportable-entity-count entities)}))
+ {}))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/stats_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/stats_test.clj
new file mode 100644
index 00000000000..029d6105b7c
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/stats_test.clj
@@ -0,0 +1,126 @@
+(ns metabase-enterprise.remote-sync.stats-test
+ (:require
+ [clojure.string :as str]
+ [clojure.test :refer :all]
+ [metabase-enterprise.remote-sync.stats :as stats]
+ [metabase.analytics.stats :as analytics.stats]
+ [metabase.collections.models.collection :as collection]
+ [metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
+ [metabase.test :as mt]
+ [metabase.util :as u]))
+
+(set! *warn-on-reflection* true)
+
+(defmacro ^:private with-remote-sync
+ [& body]
+ `(mt/with-empty-h2-app-db!
+ (mt/with-premium-features #{:remote-sync :transforms-basic}
+ (mt/with-temporary-setting-values [remote-sync-url "file:///tmp/remote-sync-repo"
+ remote-sync-transforms true]
+ ~@body))))
+
+(deftest card-counts-test
+ (with-remote-sync
+ (mt/with-temp [:model/Collection {synced-id :id} {:is_remote_synced true}
+ :model/Collection {child-id :id} {:is_remote_synced true :location (str "/" synced-id "/")}
+ :model/Collection {archived-id :id} {:is_remote_synced true :archived true}
+ :model/Collection {local-id :id} {}
+ :model/Card {source-id :id} {:collection_id local-id :type :model}
+ :model/Card _ {:collection_id synced-id :type :question
+ :dataset_query (let [mp (mt/metadata-provider)]
+ (lib/query mp (lib.metadata/card mp source-id)))}
+ :model/Card _ {:collection_id synced-id :type :question}
+ :model/Card _ {:collection_id child-id :type :metric}
+ :model/Card _ {:collection_id synced-id :type :model :archived true}
+ :model/Card _ {:collection_id archived-id :type :model}
+ :model/Card _ {:collection_id local-id :type :question}]
+ (is (=? {:remote-sync-card-count 4
+ :remote-sync-model-count 1
+ :remote-sync-question-count 2
+ :remote-sync-metric-count 1}
+ (stats/remote-sync-stats))))))
+
+(deftest table-counts-test
+ (with-remote-sync
+ (mt/with-temp [:model/Collection {coll-id :id} {:is_remote_synced true :type "library-data"}
+ :model/Database {db-id :id} {}
+ :model/Table {published-id :id} {:db_id db-id :is_published true :collection_id coll-id}
+ :model/Table {unpublished-id :id} {:db_id db-id :is_published false :collection_id coll-id}
+ :model/Field {field-id :id} {:table_id published-id}
+ :model/Field {other-id :id} {:table_id published-id}
+ :model/TableUserSettings _ {:table_id published-id :display_name "Renamed"}
+ :model/TableUserSettings _ {:table_id unpublished-id :display_name "Renamed"}
+ :model/FieldUserSettings _ {:field_id field-id :description "Curated" :description_set true}
+ :model/Dimension _ {:field_id other-id :name "Other" :type :internal}
+ :model/Measure _ {:table_id published-id}
+ :model/Measure _ {:table_id published-id :archived true}
+ :model/Measure _ {:table_id unpublished-id}
+ :model/Segment _ {:table_id published-id}
+ :model/Segment _ {:table_id unpublished-id}]
+ (is (=? {:remote-sync-table-user-settings-count 1
+ :remote-sync-field-user-settings-count 1
+ :remote-sync-dimension-count 1
+ :remote-sync-measure-count 1
+ :remote-sync-segment-count 1}
+ (stats/remote-sync-stats))))))
+
+(deftest content-counts-test
+ (with-remote-sync
+ (mt/with-temp [:model/Collection _ {:type "library" :is_remote_synced true}
+ :model/Collection {id :id} {:is_remote_synced true}
+ :model/Dashboard _ {:collection_id id}
+ :model/Document _ {:collection_id id}
+ :model/Timeline _ {:collection_id id}
+ :model/Action _ {:type :query :name "Library action" :model_id nil}
+ :model/NativeQuerySnippet _ {}
+ :model/Glossary _ {:term "Term" :definition "A definition"}
+ :model/DataApp _ {:name "app" :display_name "App" :bundle_path "app.js"}]
+ (is (=? {:remote-sync-action-count 1
+ :remote-sync-dashboard-count 1
+ :remote-sync-data-app-count 1
+ :remote-sync-document-count 1
+ :remote-sync-glossary-entry-count 1
+ :remote-sync-snippet-count 1
+ :remote-sync-timeline-count 1}
+ (stats/remote-sync-stats))))))
+
+(deftest content-total-test
+ (with-remote-sync
+ (mt/with-temp [:model/Collection {id :id} {:is_remote_synced true}
+ :model/Dashboard _ {:collection_id id}]
+ (is (=? {:remote-sync-content-count 2}
+ (stats/remote-sync-stats))))))
+
+(deftest transform-counts-test
+ (with-remote-sync
+ (mt/with-temp [:model/Collection {folder-id :id} {:namespace collection/transforms-ns}
+ :model/Collection {archived-id :id} {:namespace collection/transforms-ns :archived true}
+ :model/Transform _ {}
+ :model/Transform _ {:collection_id folder-id}
+ :model/Transform _ {:collection_id archived-id}]
+ (is (=? {:remote-sync-transform-count 2}
+ (stats/remote-sync-stats))))))
+
+(deftest disabled-sync-test
+ (mt/with-empty-h2-app-db!
+ (mt/with-premium-features #{:remote-sync}
+ (mt/with-temp [:model/Collection {id :id} {:is_remote_synced true}
+ :model/Card _ {:collection_id id}]
+ (is (= {} (stats/remote-sync-stats)))))))
+
+(deftest missing-feature-test
+ (mt/with-empty-h2-app-db!
+ (mt/with-premium-features #{}
+ (mt/with-temporary-setting-values [remote-sync-url "file:///tmp/remote-sync-repo"]
+ (mt/with-temp [:model/Collection {id :id} {:is_remote_synced true}
+ :model/Card _ {:collection_id id}]
+ (is (= {} (stats/remote-sync-stats))))))))
+
+(deftest snowplow-metrics-test
+ (with-remote-sync
+ (is (= (set (map (comp name u/->snake_case_en) (keys (stats/remote-sync-stats))))
+ (into #{}
+ (comp (map #(get % "name")) (filter #(str/starts-with? % "remote_sync_")))
+ (get (#'analytics.stats/snowplow-anonymous-usage-stats (analytics.stats/legacy-anonymous-usage-stats))
+ "metrics"))))))
diff --git a/src/metabase/analytics/stats.clj b/src/metabase/analytics/stats.clj
index a69de3881e1..c388cf459d2 100644
--- a/src/metabase/analytics/stats.clj
+++ b/src/metabase/analytics/stats.clj
@@ -583,6 +583,17 @@
{:transforms (analytics.db/transform-count)
:transform_runs_last_24h (analytics.db/transform-run-count-since one-day-ago)}))
+(defenterprise remote-sync-stats
+ "Counts of the content a full Git sync export serializes, by model and in total, empty without Git sync."
+ metabase-enterprise.remote-sync.stats
+ []
+ {})
+
+(defn- remote-sync-metrics
+ "Returns Git-synced content counts for the Snowplow stats ping."
+ []
+ (update-keys (remote-sync-stats) u/->snake_case_en))
+
(defn- ->snowplow-metric-info
"Collects Snowplow metrics data that is not in the legacy stats format. Also clears entity id translation count."
[]
@@ -595,7 +606,8 @@
:pivot_tables (analytics.db/unarchived-pivot-table-count)
:query_executions_last_24h (analytics.db/query-execution-count-since one-day-ago)
:entity_id_translations_last_24h total-translation-count}
- (transform-metrics))))
+ (transform-metrics)
+ (remote-sync-metrics))))
(def ^:private string-keyed-int-histogram
[:map-of :string :int])
@@ -750,7 +762,25 @@
[:query_executions_last_24h :int]
[:entity_id_translations_last_24h :int]
[:transforms :int]
- [:transform_runs_last_24h :int]]]
+ [:transform_runs_last_24h :int]
+ [:remote_sync_action_count {:optional true} :int]
+ [:remote_sync_card_count {:optional true} :int]
+ [:remote_sync_content_count {:optional true} :int]
+ [:remote_sync_dashboard_count {:optional true} :int]
+ [:remote_sync_data_app_count {:optional true} :int]
+ [:remote_sync_dimension_count {:optional true} :int]
+ [:remote_sync_document_count {:optional true} :int]
+ [:remote_sync_field_user_settings_count {:optional true} :int]
+ [:remote_sync_glossary_entry_count {:optional true} :int]
+ [:remote_sync_measure_count {:optional true} :int]
+ [:remote_sync_metric_count {:optional true} :int]
+ [:remote_sync_model_count {:optional true} :int]
+ [:remote_sync_question_count {:optional true} :int]
+ [:remote_sync_segment_count {:optional true} :int]
+ [:remote_sync_snippet_count {:optional true} :int]
+ [:remote_sync_table_user_settings_count {:optional true} :int]
+ [:remote_sync_timeline_count {:optional true} :int]
+ [:remote_sync_transform_count {:optional true} :int]]]
(mapv
(fn [[k v tags]]
(assert (every? string? tags) "Tags must be strings in snowplow metrics.")
@@ -787,6 +817,24 @@
[:public_questions_with_params (get-in stats [:stats :question :public :with_params] 0) #{"questions"}]
[:query_builder_questions (get-in stats [:stats :question :questions :total] 0) #{"questions"}]
[:query_executions_last_24h (:query_executions_last_24h metric-info 0) #{"query_executions"}]
+ [:remote_sync_action_count (:remote_sync_action_count metric-info 0) #{"remote_sync"}]
+ [:remote_sync_card_count (:remote_sync_card_count metric-info 0) #{"remote_sync"}]
+ [:remote_sync_content_count (:remote_sync_content_count metric-info 0) #{"remote_sync"}]
+ [:remote_sync_dashboard_count (:remote_sync_dashboard_count metric-info 0) #{"remote_sync"}]
+ [:remote_sync_data_app_count (:remote_sync_data_app_count metric-info 0) #{"remote_sync"}]
+ [:remote_sync_dimension_count (:remote_sync_dimension_count metric-info 0) #{"remote_sync"}]
+ [:remote_sync_document_count (:remote_sync_document_count metric-info 0) #{"remote_sync"}]
+ [:remote_sync_field_user_settings_count (:remote_sync_field_user_settings_count metric-info 0) #{"remote_sync"}]
+ [:remote_sync_glossary_entry_count (:remote_sync_glossary_entry_count metric-info 0) #{"remote_sync"}]
+ [:remote_sync_measure_count (:remote_sync_measure_count metric-info 0) #{"remote_sync"}]
+ [:remote_sync_metric_count (:remote_sync_metric_count metric-info 0) #{"remote_sync"}]
+ [:remote_sync_model_count (:remote_sync_model_count metric-info 0) #{"remote_sync"}]
+ [:remote_sync_question_count (:remote_sync_question_count metric-info 0) #{"remote_sync"}]
+ [:remote_sync_segment_count (:remote_sync_segment_count metric-info 0) #{"remote_sync"}]
+ [:remote_sync_snippet_count (:remote_sync_snippet_count metric-info 0) #{"remote_sync"}]
+ [:remote_sync_table_user_settings_count (:remote_sync_table_user_settings_count metric-info 0) #{"remote_sync"}]
+ [:remote_sync_timeline_count (:remote_sync_timeline_count metric-info 0) #{"remote_sync"}]
+ [:remote_sync_transform_count (:remote_sync_transform_count metric-info 0) #{"remote_sync"}]
[:questions (get-in stats [:stats :question :questions :total] 0) #{"questions"}]
[:questions_with_params (get-in stats [:stats :question :questions :with_params] 0) #{"questions"}]
[:segments (get-in stats [:stats :segment :segments] 0) #{"segments"}]