The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Add Git-synced content counts to the anonymous stats ping (#84572)
dpsutton65afc667cf07
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/db.cljindex b036956cac0..ed30c2c1298 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj@@ -297,6 +297,11 @@[card-ids :- [:sequential ::lib.schema.id/card]](t2/select [:model/Card :id :type :display] :id [:in card-ids]))+(mu/defn card-counts-by-type :- [:map-of :keyword :int]+ "The number of Cards with `card-ids`, by `:type`."+ [card-ids :- [:sequential ::lib.schema.id/card]]+ (t2/select-fn->fn :type :count [:model/Card :type [:%count.* :count]] :id [:in card-ids] {:group-by [:type]}))+(mu/defn published-table-ids :- [:set ::lib.schema.id/table]"The ids of the Tables published in the Collections with `collection-ids`."[collection-ids :- [:sequential ::lib.schema.id/collection]]diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/stats.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/stats.cljnew file mode 100644index 00000000000..5b4b8f9e0e0--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/stats.clj@@ -0,0 +1,47 @@+(ns metabase-enterprise.remote-sync.stats+ "Counts of Git-synced content for the anonymous stats ping."+ (:require+ [metabase-enterprise.remote-sync.db :as remote-sync.db]+ [metabase-enterprise.remote-sync.impl :as impl]+ [metabase-enterprise.remote-sync.settings :as settings]+ [metabase-enterprise.remote-sync.spec :as spec]+ [metabase.premium-features.core :refer [defenterprise]]))++(set! *warn-on-reflection* true)++(def ^:private model-count-keys+ {"Action" :remote-sync-action-count+ "Dashboard" :remote-sync-dashboard-count+ "DataApp" :remote-sync-data-app-count+ "Dimension" :remote-sync-dimension-count+ "Document" :remote-sync-document-count+ "FieldUserSettings" :remote-sync-field-user-settings-count+ "Glossary" :remote-sync-glossary-entry-count+ "Measure" :remote-sync-measure-count+ "NativeQuerySnippet" :remote-sync-snippet-count+ "Segment" :remote-sync-segment-count+ "TableUserSettings" :remote-sync-table-user-settings-count+ "Timeline" :remote-sync-timeline-count+ "Transform" :remote-sync-transform-count})++(defn- card-type-counts+ [card-ids]+ (let [counts (transduce (map #(remote-sync.db/card-counts-by-type (vec %)))+ (partial merge-with +)+ {}+ (partition-all impl/app-db-batch-size card-ids))]+ {:remote-sync-card-count (reduce + 0 (vals counts))+ :remote-sync-model-count (get counts :model 0)+ :remote-sync-question-count (get counts :question 0)+ :remote-sync-metric-count (get counts :metric 0)}))++(defenterprise remote-sync-stats+ "Counts of the content a full Git sync export serializes, by model and in total, empty without Git sync."+ :feature :remote-sync+ []+ (if (settings/remote-sync-enabled)+ (let [entities (spec/exportable-entities)]+ (merge (into {} (map (fn [[model k]] [k (count (get entities model))])) model-count-keys)+ (card-type-counts (get entities "Card"))+ {:remote-sync-content-count (spec/exportable-entity-count entities)}))+ {}))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/stats_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/stats_test.cljnew file mode 100644index 00000000000..029d6105b7c--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/stats_test.clj@@ -0,0 +1,126 @@+(ns metabase-enterprise.remote-sync.stats-test+ (:require+ [clojure.string :as str]+ [clojure.test :refer :all]+ [metabase-enterprise.remote-sync.stats :as stats]+ [metabase.analytics.stats :as analytics.stats]+ [metabase.collections.models.collection :as collection]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.test :as mt]+ [metabase.util :as u]))++(set! *warn-on-reflection* true)++(defmacro ^:private with-remote-sync+ [& body]+ `(mt/with-empty-h2-app-db!+ (mt/with-premium-features #{:remote-sync :transforms-basic}+ (mt/with-temporary-setting-values [remote-sync-url "file:///tmp/remote-sync-repo"+ remote-sync-transforms true]+ ~@body))))++(deftest card-counts-test+ (with-remote-sync+ (mt/with-temp [:model/Collection {synced-id :id} {:is_remote_synced true}+ :model/Collection {child-id :id} {:is_remote_synced true :location (str "/" synced-id "/")}+ :model/Collection {archived-id :id} {:is_remote_synced true :archived true}+ :model/Collection {local-id :id} {}+ :model/Card {source-id :id} {:collection_id local-id :type :model}+ :model/Card _ {:collection_id synced-id :type :question+ :dataset_query (let [mp (mt/metadata-provider)]+ (lib/query mp (lib.metadata/card mp source-id)))}+ :model/Card _ {:collection_id synced-id :type :question}+ :model/Card _ {:collection_id child-id :type :metric}+ :model/Card _ {:collection_id synced-id :type :model :archived true}+ :model/Card _ {:collection_id archived-id :type :model}+ :model/Card _ {:collection_id local-id :type :question}]+ (is (=? {:remote-sync-card-count 4+ :remote-sync-model-count 1+ :remote-sync-question-count 2+ :remote-sync-metric-count 1}+ (stats/remote-sync-stats))))))++(deftest table-counts-test+ (with-remote-sync+ (mt/with-temp [:model/Collection {coll-id :id} {:is_remote_synced true :type "library-data"}+ :model/Database {db-id :id} {}+ :model/Table {published-id :id} {:db_id db-id :is_published true :collection_id coll-id}+ :model/Table {unpublished-id :id} {:db_id db-id :is_published false :collection_id coll-id}+ :model/Field {field-id :id} {:table_id published-id}+ :model/Field {other-id :id} {:table_id published-id}+ :model/TableUserSettings _ {:table_id published-id :display_name "Renamed"}+ :model/TableUserSettings _ {:table_id unpublished-id :display_name "Renamed"}+ :model/FieldUserSettings _ {:field_id field-id :description "Curated" :description_set true}+ :model/Dimension _ {:field_id other-id :name "Other" :type :internal}+ :model/Measure _ {:table_id published-id}+ :model/Measure _ {:table_id published-id :archived true}+ :model/Measure _ {:table_id unpublished-id}+ :model/Segment _ {:table_id published-id}+ :model/Segment _ {:table_id unpublished-id}]+ (is (=? {:remote-sync-table-user-settings-count 1+ :remote-sync-field-user-settings-count 1+ :remote-sync-dimension-count 1+ :remote-sync-measure-count 1+ :remote-sync-segment-count 1}+ (stats/remote-sync-stats))))))++(deftest content-counts-test+ (with-remote-sync+ (mt/with-temp [:model/Collection _ {:type "library" :is_remote_synced true}+ :model/Collection {id :id} {:is_remote_synced true}+ :model/Dashboard _ {:collection_id id}+ :model/Document _ {:collection_id id}+ :model/Timeline _ {:collection_id id}+ :model/Action _ {:type :query :name "Library action" :model_id nil}+ :model/NativeQuerySnippet _ {}+ :model/Glossary _ {:term "Term" :definition "A definition"}+ :model/DataApp _ {:name "app" :display_name "App" :bundle_path "app.js"}]+ (is (=? {:remote-sync-action-count 1+ :remote-sync-dashboard-count 1+ :remote-sync-data-app-count 1+ :remote-sync-document-count 1+ :remote-sync-glossary-entry-count 1+ :remote-sync-snippet-count 1+ :remote-sync-timeline-count 1}+ (stats/remote-sync-stats))))))++(deftest content-total-test+ (with-remote-sync+ (mt/with-temp [:model/Collection {id :id} {:is_remote_synced true}+ :model/Dashboard _ {:collection_id id}]+ (is (=? {:remote-sync-content-count 2}+ (stats/remote-sync-stats))))))++(deftest transform-counts-test+ (with-remote-sync+ (mt/with-temp [:model/Collection {folder-id :id} {:namespace collection/transforms-ns}+ :model/Collection {archived-id :id} {:namespace collection/transforms-ns :archived true}+ :model/Transform _ {}+ :model/Transform _ {:collection_id folder-id}+ :model/Transform _ {:collection_id archived-id}]+ (is (=? {:remote-sync-transform-count 2}+ (stats/remote-sync-stats))))))++(deftest disabled-sync-test+ (mt/with-empty-h2-app-db!+ (mt/with-premium-features #{:remote-sync}+ (mt/with-temp [:model/Collection {id :id} {:is_remote_synced true}+ :model/Card _ {:collection_id id}]+ (is (= {} (stats/remote-sync-stats)))))))++(deftest missing-feature-test+ (mt/with-empty-h2-app-db!+ (mt/with-premium-features #{}+ (mt/with-temporary-setting-values [remote-sync-url "file:///tmp/remote-sync-repo"]+ (mt/with-temp [:model/Collection {id :id} {:is_remote_synced true}+ :model/Card _ {:collection_id id}]+ (is (= {} (stats/remote-sync-stats))))))))++(deftest snowplow-metrics-test+ (with-remote-sync+ (is (= (set (map (comp name u/->snake_case_en) (keys (stats/remote-sync-stats))))+ (into #{}+ (comp (map #(get % "name")) (filter #(str/starts-with? % "remote_sync_")))+ (get (#'analytics.stats/snowplow-anonymous-usage-stats (analytics.stats/legacy-anonymous-usage-stats))+ "metrics"))))))diff --git a/src/metabase/analytics/stats.clj b/src/metabase/analytics/stats.cljindex a69de3881e1..c388cf459d2 100644--- a/src/metabase/analytics/stats.clj+++ b/src/metabase/analytics/stats.clj@@ -583,6 +583,17 @@{:transforms (analytics.db/transform-count):transform_runs_last_24h (analytics.db/transform-run-count-since one-day-ago)}))+(defenterprise remote-sync-stats+ "Counts of the content a full Git sync export serializes, by model and in total, empty without Git sync."+ metabase-enterprise.remote-sync.stats+ []+ {})++(defn- remote-sync-metrics+ "Returns Git-synced content counts for the Snowplow stats ping."+ []+ (update-keys (remote-sync-stats) u/->snake_case_en))+(defn- ->snowplow-metric-info"Collects Snowplow metrics data that is not in the legacy stats format. Also clears entity id translation count."[]@@ -595,7 +606,8 @@:pivot_tables (analytics.db/unarchived-pivot-table-count):query_executions_last_24h (analytics.db/query-execution-count-since one-day-ago):entity_id_translations_last_24h total-translation-count}- (transform-metrics))))+ (transform-metrics)+ (remote-sync-metrics))))(def ^:private string-keyed-int-histogram[:map-of :string :int])@@ -750,7 +762,25 @@[:query_executions_last_24h :int][:entity_id_translations_last_24h :int][:transforms :int]- [:transform_runs_last_24h :int]]]+ [:transform_runs_last_24h :int]+ [:remote_sync_action_count {:optional true} :int]+ [:remote_sync_card_count {:optional true} :int]+ [:remote_sync_content_count {:optional true} :int]+ [:remote_sync_dashboard_count {:optional true} :int]+ [:remote_sync_data_app_count {:optional true} :int]+ [:remote_sync_dimension_count {:optional true} :int]+ [:remote_sync_document_count {:optional true} :int]+ [:remote_sync_field_user_settings_count {:optional true} :int]+ [:remote_sync_glossary_entry_count {:optional true} :int]+ [:remote_sync_measure_count {:optional true} :int]+ [:remote_sync_metric_count {:optional true} :int]+ [:remote_sync_model_count {:optional true} :int]+ [:remote_sync_question_count {:optional true} :int]+ [:remote_sync_segment_count {:optional true} :int]+ [:remote_sync_snippet_count {:optional true} :int]+ [:remote_sync_table_user_settings_count {:optional true} :int]+ [:remote_sync_timeline_count {:optional true} :int]+ [:remote_sync_transform_count {:optional true} :int]]](mapv(fn [[k v tags]](assert (every? string? tags) "Tags must be strings in snowplow metrics.")@@ -787,6 +817,24 @@[:public_questions_with_params (get-in stats [:stats :question :public :with_params] 0) #{"questions"}][:query_builder_questions (get-in stats [:stats :question :questions :total] 0) #{"questions"}][:query_executions_last_24h (:query_executions_last_24h metric-info 0) #{"query_executions"}]+ [:remote_sync_action_count (:remote_sync_action_count metric-info 0) #{"remote_sync"}]+ [:remote_sync_card_count (:remote_sync_card_count metric-info 0) #{"remote_sync"}]+ [:remote_sync_content_count (:remote_sync_content_count metric-info 0) #{"remote_sync"}]+ [:remote_sync_dashboard_count (:remote_sync_dashboard_count metric-info 0) #{"remote_sync"}]+ [:remote_sync_data_app_count (:remote_sync_data_app_count metric-info 0) #{"remote_sync"}]+ [:remote_sync_dimension_count (:remote_sync_dimension_count metric-info 0) #{"remote_sync"}]+ [:remote_sync_document_count (:remote_sync_document_count metric-info 0) #{"remote_sync"}]+ [:remote_sync_field_user_settings_count (:remote_sync_field_user_settings_count metric-info 0) #{"remote_sync"}]+ [:remote_sync_glossary_entry_count (:remote_sync_glossary_entry_count metric-info 0) #{"remote_sync"}]+ [:remote_sync_measure_count (:remote_sync_measure_count metric-info 0) #{"remote_sync"}]+ [:remote_sync_metric_count (:remote_sync_metric_count metric-info 0) #{"remote_sync"}]+ [:remote_sync_model_count (:remote_sync_model_count metric-info 0) #{"remote_sync"}]+ [:remote_sync_question_count (:remote_sync_question_count metric-info 0) #{"remote_sync"}]+ [:remote_sync_segment_count (:remote_sync_segment_count metric-info 0) #{"remote_sync"}]+ [:remote_sync_snippet_count (:remote_sync_snippet_count metric-info 0) #{"remote_sync"}]+ [:remote_sync_table_user_settings_count (:remote_sync_table_user_settings_count metric-info 0) #{"remote_sync"}]+ [:remote_sync_timeline_count (:remote_sync_timeline_count metric-info 0) #{"remote_sync"}]+ [:remote_sync_transform_count (:remote_sync_transform_count metric-info 0) #{"remote_sync"}][:questions (get-in stats [:stats :question :questions :total] 0) #{"questions"}][:questions_with_params (get-in stats [:stats :question :questions :with_params] 0) #{"questions"}][:segments (get-in stats [:stats :segment :segments] 0) #{"segments"}]