The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Introduce experiments (#72794)
metamben190b23d30fe7
diff --git a/src/metabase/analytics/experiment.cljc b/src/metabase/analytics/experiment.cljcnew file mode 100644index 00000000000..42ee5c6bc2b--- /dev/null+++ b/src/metabase/analytics/experiment.cljc@@ -0,0 +1,35 @@+(ns metabase.analytics.experiment+ "Default experiment report function backed by metabase.analytics-interface.core.+ Requiring this namespace wires `report!` as the default report fn."+ (:require+ [metabase.analytics-interface.core :as analytics]+ [metabase.util.experiment :as experiment]))++(defn report!+ "Report an experiment result via the analytics interface (Prometheus on JVM, batched POST on CLJS).++ Expects a map with keys:+ :name - keyword identifying the experiment+ :match? - whether control and candidate results matched+ :control-duration-ns - nanoseconds the control took+ :candidate-duration-ns - nanoseconds the candidate took+ :control-outcome - {:result v} or {:error t}+ :candidate-outcome - {:result v} or {:error t}"+ [{:keys [name match? control-duration-ns candidate-duration-ns candidate-outcome]}]+ (let [labels {:experiment (cond-> (str name)+ (keyword? name) (subs 1))}+ error? (contains? candidate-outcome :error)+ outcome-key (cond+ match? :experiment/matches-total+ error? :experiment/errors-total+ :else :experiment/mismatches-total)+ candidate-duration-key (if error?+ :experiment/candidate-error-duration-ms+ :experiment/candidate-duration-ms)]+ (analytics/inc! :experiment/runs-total labels)+ (analytics/inc! outcome-key labels)+ (analytics/inc! :experiment/control-duration-ms labels (/ control-duration-ns 1e6))+ (analytics/inc! candidate-duration-key labels (/ candidate-duration-ns 1e6))))++;; Wire as the default report fn. Requiring this namespace is sufficient.+(experiment/set-default-report-fn! report!)diff --git a/src/metabase/analytics/prometheus.clj b/src/metabase/analytics/prometheus.cljindex 86655cba059..671368f5a89 100644--- a/src/metabase/analytics/prometheus.clj+++ b/src/metabase/analytics/prometheus.clj@@ -646,7 +646,30 @@{:description "Number of frontend analytics events dropped before being POSTed to the backend, because the client-side buffer was full."})(prometheus/counter :metabase-export/errors{:description "Number of errors during data export."- :labels [:format]})])+ :labels [:format]})++ ;; experiment metrics+ (prometheus/counter :experiment/runs-total+ {:description "Number of experiment candidate runs."+ :labels [:experiment]})+ (prometheus/counter :experiment/matches-total+ {:description "Number of experiment runs where control and candidate matched."+ :labels [:experiment]})+ (prometheus/counter :experiment/mismatches-total+ {:description "Number of experiment runs where control and candidate differed."+ :labels [:experiment]})+ (prometheus/counter :experiment/errors-total+ {:description "Number of experiment runs where the candidate threw."+ :labels [:experiment]})+ (prometheus/counter :experiment/control-duration-ms+ {:description "Cumulative duration in milliseconds of experiment control code path."+ :labels [:experiment]})+ (prometheus/counter :experiment/candidate-duration-ms+ {:description "Cumulative duration in milliseconds of experiment candidate code path."+ :labels [:experiment]})+ (prometheus/counter :experiment/candidate-error-duration-ms+ {:description "Cumulative duration in milliseconds of experiment candidate code path when it threw."+ :labels [:experiment]})])(defn- quartz-collectors[]diff --git a/src/metabase/lib/js.cljs b/src/metabase/lib/js.cljsindex 2d99322c848..572c11f832c 100644--- a/src/metabase/lib/js.cljs+++ b/src/metabase/lib/js.cljs@@ -61,6 +61,7 @@[goog.object :as gobject][medley.core :as m][metabase.analytics-interface.core :as analytics.interface]+ [metabase.analytics.experiment][metabase.analytics.impl]^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.normalize :as mbql.normalize][metabase.lib.aggregation :as lib.aggregation]@@ -98,8 +99,10 @@;;; This ensures that all of metabase.lib.* is loaded, so all the `defmethod`s are properly registered.;;; metabase.analytics.impl registers the CLJS reporter for [[metabase.analytics-interface.core]].+;;; metabase.analytics.experiment wires the default experiment report fn.(comment lib.core/keep-me+ metabase.analytics.experiment/keep-memetabase.analytics.impl/keep-me);; Expose for E2E testingdiff --git a/src/metabase/util/experiment.cljc b/src/metabase/util/experiment.cljcnew file mode 100644index 00000000000..76f6a1cd7f0--- /dev/null+++ b/src/metabase/util/experiment.cljc@@ -0,0 +1,173 @@+(ns metabase.util.experiment+ "Framework for running experimental code paths alongside+ production code, comparing results, and reporting metrics.++ Usage:+ (experiment {:name :my-experiment :min-interval-ms 5000}+ (production-code)+ (experimental-code))++ Always returns the result of the production (control) code path.+ The candidate is only run when the global kill switch is on and+ the time-based throttle allows it."+ (:require+ [metabase.util.log :as log]+ [net.cgrand.macrovich :as macros])+ #?(:cljs (:require-macros [metabase.util.experiment])))++;;; ------------------------------------------------ Configuration ------------------------------------------------++(defonce ^{:private true+ :doc "Atom holding a zero-arg function that returns whether experiments are enabled.+ Defaults to (constantly true). On JVM, wired to the setting getter at startup.+ On CLJS, wired to read from bootstrap data."}+ experiments-enabled-fn+ (atom (constantly true)))++(defn experiments-enabled?+ "Returns true if experiments are globally enabled. Checks the setting on every call."+ []+ (@experiments-enabled-fn))++(defn set-experiments-enabled-fn!+ "Set the function that determines whether experiments are enabled.+ Called once at application startup to wire in the setting."+ [f]+ (reset! experiments-enabled-fn f))++(defonce ^{:private true+ :doc "Atom holding the default report function. Set at application startup via [[set-default-report-fn!]].+ Nil means no reporting. Can be overridden per-experiment via :report-fn in the opts map."}+ default-report-fn (atom nil))++(defn set-default-report-fn!+ "Set the default report function for experiment results."+ [f]+ (reset! default-report-fn f))++(def ^:dynamic *sync?*+ "When true, forces synchronous candidate execution on JVM. For testing only."+ false)++;;; -------------------------------------------------- Throttling -------------------------------------------------++(defonce ^:private last-run-times (atom {}))++(defn- current-time-ms []+ #?(:clj (System/currentTimeMillis)+ :cljs (js/Date.now)))++(defn- throttle-allows?+ "Returns true if enough time has elapsed since the last run of this experiment."+ [experiment-name min-interval-ms]+ (let [now (current-time-ms)+ last-run (get @last-run-times experiment-name 0)]+ (when (>= (- now last-run) min-interval-ms)+ (swap! last-run-times assoc experiment-name now)+ true)))++;;; --------------------------------------------- Outcome comparison ----------------------------------------------++(defn- outcomes-match?+ "Compare control and candidate outcomes. Both are {:result v} or {:error t}.+ Results match via comparator-fn. Errors match if same type, ex-message, and ex-data."+ [comparator-fn control candidate]+ (try+ (cond+ (and (contains? control :result) (contains? candidate :result))+ (comparator-fn (:result control) (:result candidate))++ (and (contains? control :error) (contains? candidate :error))+ (let [ce (:error control)+ ca (:error candidate)]+ (and (= (type ce) (type ca))+ (= (ex-message ce) (ex-message ca))+ (= (ex-data ce) (ex-data ca))))++ :else false)+ (catch #?(:clj Throwable :cljs :default) _+ false)))++;;; ------------------------------------------- Candidate execution -----------------------------------------------++(defn nanotime+ "Returns current time in nanoseconds. Cross-platform. Public because the `experiment` macro+ expands calls to this in the caller's namespace."+ []+ #?(:clj (System/nanoTime)+ :cljs (* 1000000 (js/performance.now))))++(defn run-experiment*+ "Runtime implementation. Called by the `experiment` macro. Do not call directly.+ Checks the global kill switch and throttle, and if both allow, runs the candidate."+ [{:keys [name min-interval-ms comparator-fn report-fn]} control-outcome control-duration-ns candidate-thunk]+ (when (and (experiments-enabled?)+ (throttle-allows? name (or min-interval-ms 1000)))+ (let [comparator-fn (or comparator-fn =)+ report-fn (or report-fn @default-report-fn)+ run-fn+ (fn []+ (let [start (nanotime)+ candidate-outcome (try+ {:result (candidate-thunk)}+ (catch #?(:clj Throwable :cljs :default) t+ {:error t}))+ end (nanotime)+ duration (- end start)+ match? (outcomes-match? comparator-fn control-outcome candidate-outcome)]+ (when-not match?+ (if (contains? candidate-outcome :error)+ (log/warnf "Experiment %s: candidate threw %s" name (str (:error candidate-outcome)))+ (log/warnf "Experiment %s: result MISMATCH" name)))+ (when report-fn+ (report-fn {:name name+ :match? match?+ :control-duration-ns control-duration-ns+ :candidate-duration-ns duration+ :control-outcome control-outcome+ :candidate-outcome candidate-outcome}))))]+ ;; NOTE: `future-call` conveys thread bindings automatically via `binding-conveyor-fn`.+ ;; If switching to a custom executor, capture bindings with `(get-thread-bindings)` before+ ;; submission, then restore them with `(push-thread-bindings)` / `(pop-thread-bindings)`+ ;; around the body in the worker thread.+ #?(:clj (if *sync?* (run-fn) (future-call run-fn))+ :cljs (run-fn)))))++;;; --------------------------------------------------- Macro ----------------------------------------------------++(defmacro experiment+ "Run an experiment comparing control and candidate code paths.++ Always returns the control result (or re-throws the control exception).+ The candidate is only run when the global kill switch is on and+ the time-based throttle allows it.++ Options map:+ :name - keyword identifying this experiment (required)+ :min-interval-ms - minimum ms between candidate runs (default 1000)+ :comparator-fn - 2-arg fn for comparing results (default =)+ :report-fn - 1-arg fn called with result map (default: the fn set via set-default-report-fn!)++ Usage:+ (experiment {:name :my-experiment}+ (control-expression)+ (candidate-expression))"+ {:style/indent 1}+ [opts control candidate]+ (assert (:name opts) ":name is required for experiment")+ `(let [start# (nanotime)+ control-outcome# (try+ {:result ~control}+ (catch ~(macros/case :clj 'Throwable :cljs ':default) t#+ {:error t#}))+ duration# (- (nanotime) start#)]+ (run-experiment*+ ~opts+ control-outcome#+ duration#+ ~(macros/case+ :clj `(^:once fn* [] ~candidate)+ :cljs `(fn [] ~candidate)))+ (if (contains? control-outcome# :error)+ (throw (:error control-outcome#))+ (:result control-outcome#))))diff --git a/src/metabase/util/experiment/settings.clj b/src/metabase/util/experiment/settings.cljnew file mode 100644index 00000000000..2b9b4c3f416--- /dev/null+++ b/src/metabase/util/experiment/settings.clj@@ -0,0 +1,18 @@+(ns metabase.util.experiment.settings+ (:require+ [metabase.settings.core :refer [defsetting]]+ [metabase.util.experiment :as experiment]+ [metabase.util.i18n :refer [deferred-tru]]))++#_{:clj-kondo/ignore [:metabase/defsetting-namespace]}+(defsetting experiments-enabled+ (deferred-tru "Enable or disable all code experiments. When disabled, only the production code path runs.")+ :type :boolean+ :default false+ :visibility :admin+ :export? false+ :audit :getter)++;; Wire the setting into the experiment system. The function is called on every experiment+;; invocation, so toggling the setting in the admin panel takes effect immediately.+(experiment/set-experiments-enabled-fn! experiments-enabled)diff --git a/test/metabase/analytics/experiment_test.cljc b/test/metabase/analytics/experiment_test.cljcnew file mode 100644index 00000000000..80ca4906cf2--- /dev/null+++ b/test/metabase/analytics/experiment_test.cljc@@ -0,0 +1,120 @@+(ns metabase.analytics.experiment-test+ (:require+ [clojure.test :refer [deftest is testing]]+ [metabase.analytics-interface.core :as analytics.interface]+ [metabase.analytics.experiment :as analytics.experiment]+ [metabase.util.experiment :as experiment]))++(defn- make-test-reporter [calls]+ (reify analytics.interface/Reporter+ (-inc! [_ metric labels amount]+ (swap! calls conj {:op :inc! :metric metric :labels labels :amount amount}))+ (-dec-gauge! [_ metric labels amount]+ (swap! calls conj {:op :dec-gauge! :metric metric :labels labels :amount amount}))+ (-set-gauge! [_ metric labels amount]+ (swap! calls conj {:op :set-gauge! :metric metric :labels labels :amount amount}))+ (-observe! [_ metric labels amount]+ (swap! calls conj {:op :observe! :metric metric :labels labels :amount amount}))+ (-clear! [_ metric]+ (swap! calls conj {:op :clear! :metric metric}))))++(defn- do-with-test-reporter! [f]+ (let [calls (atom [])+ original-reporter (analytics.interface/get-reporter)]+ (try+ (analytics.interface/set-reporter! (make-test-reporter calls))+ (f calls)+ (finally+ (analytics.interface/set-reporter! original-reporter)))))++(defn- calls-for [calls metric]+ (filterv #(= metric (:metric %)) @calls))++;;; ---------------------------------------------------- Tests ----------------------------------------------------++(deftest report-match-test+ (testing "matching result increments runs-total and matches-total, observes durations"+ (do-with-test-reporter!+ (fn [calls]+ (analytics.experiment/report!+ {:name :test/match+ :match? true+ :control-duration-ns 1000000+ :candidate-duration-ns 2000000+ :control-outcome {:result 42}+ :candidate-outcome {:result 42}})+ (let [labels {:experiment "test/match"}]+ (is (= [{:op :inc! :metric :experiment/runs-total :labels labels :amount 1}]+ (calls-for calls :experiment/runs-total)))+ (is (= [{:op :inc! :metric :experiment/matches-total :labels labels :amount 1}]+ (calls-for calls :experiment/matches-total)))+ (is (empty? (calls-for calls :experiment/mismatches-total)))+ (is (empty? (calls-for calls :experiment/errors-total)))+ (is (= [{:op :inc! :metric :experiment/control-duration-ms :labels labels :amount 1.0}]+ (calls-for calls :experiment/control-duration-ms)))+ (is (= [{:op :inc! :metric :experiment/candidate-duration-ms :labels labels :amount 2.0}]+ (calls-for calls :experiment/candidate-duration-ms))))))))++(deftest report-mismatch-test+ (testing "mismatching result increments runs-total and mismatches-total"+ (do-with-test-reporter!+ (fn [calls]+ (analytics.experiment/report!+ {:name :test/mismatch+ :match? false+ :control-duration-ns 1000000+ :candidate-duration-ns 2000000+ :control-outcome {:result :old}+ :candidate-outcome {:result :new}})+ (let [labels {:experiment "test/mismatch"}]+ (is (= [{:op :inc! :metric :experiment/runs-total :labels labels :amount 1}]+ (calls-for calls :experiment/runs-total)))+ (is (empty? (calls-for calls :experiment/matches-total)))+ (is (= [{:op :inc! :metric :experiment/mismatches-total :labels labels :amount 1}]+ (calls-for calls :experiment/mismatches-total)))+ (is (empty? (calls-for calls :experiment/errors-total))))))))++(deftest report-matching-errors-test+ (testing "both threw same exception: counted as match, not error"+ (do-with-test-reporter!+ (fn [calls]+ (let [ex (ex-info "boom" {:code 1})]+ (analytics.experiment/report!+ {:name :test/matching-error+ :match? true+ :control-duration-ns 1000000+ :candidate-duration-ns 800000+ :control-outcome {:error ex}+ :candidate-outcome {:error ex}}))+ (let [labels {:experiment "test/matching-error"}]+ (is (= [{:op :inc! :metric :experiment/matches-total :labels labels :amount 1}]+ (calls-for calls :experiment/matches-total)))+ (is (empty? (calls-for calls :experiment/errors-total)))+ (is (empty? (calls-for calls :experiment/mismatches-total))))))))++(deftest report-candidate-error-test+ (testing "candidate error (mismatch) increments runs-total and errors-total"+ (do-with-test-reporter!+ (fn [calls]+ (analytics.experiment/report!+ {:name :test/error+ :match? false+ :control-duration-ns 1000000+ :candidate-duration-ns 500000+ :control-outcome {:result :ok}+ :candidate-outcome {:error (ex-info "boom" {})}})+ (let [labels {:experiment "test/error"}]+ (is (= [{:op :inc! :metric :experiment/runs-total :labels labels :amount 1}]+ (calls-for calls :experiment/runs-total)))+ (is (= [{:op :inc! :metric :experiment/errors-total :labels labels :amount 1}]+ (calls-for calls :experiment/errors-total)))+ (is (empty? (calls-for calls :experiment/matches-total)))+ (is (empty? (calls-for calls :experiment/mismatches-total)))+ (is (empty? (calls-for calls :experiment/candidate-duration-ms)))+ (is (= [{:op :inc! :metric :experiment/candidate-error-duration-ms :labels labels :amount 0.5}]+ (calls-for calls :experiment/candidate-error-duration-ms))))))))++(deftest wiring-test+ (testing "requiring metabase.analytics.experiment installs report! as the default report fn"+ (is (= analytics.experiment/report!+ @@#'experiment/default-report-fn))))diff --git a/test/metabase/util/experiment_test.cljc b/test/metabase/util/experiment_test.cljcnew file mode 100644index 00000000000..c3cf1bfa32d--- /dev/null+++ b/test/metabase/util/experiment_test.cljc@@ -0,0 +1,243 @@+(ns metabase.util.experiment-test+ (:require+ [clojure.test :refer [deftest is testing use-fixtures]]+ [metabase.util.experiment :as experiment])+ #?(:cljs (:require-macros [metabase.util.experiment])))++(defn- capture-report-fn+ "Returns [report-fn, results-atom]. The report-fn appends each result map to the atom."+ []+ (let [results (atom [])]+ [(fn [result] (swap! results conj result))+ results]))++(use-fixtures :each+ (fn [t]+ (binding [experiment/*sync?* true]+ (t))))++;;; ------------------------------------------------ Core behavior ------------------------------------------------++(deftest control-result-returned-test+ (testing "when candidate matches"+ (is (= :control+ (experiment/experiment {:name :test/match :min-interval-ms 0}+ :control+ :control))))+ (testing "when candidate mismatches"+ (is (= :control+ (experiment/experiment {:name :test/mismatch :min-interval-ms 0}+ :control+ :candidate))))+ (testing "when candidate throws"+ (is (= :control+ (experiment/experiment {:name :test/candidate-throws :min-interval-ms 0}+ :control+ (throw (ex-info "boom" {})))))))++(deftest control-exception-rethrown-test+ (is (thrown-with-msg?+ #?(:clj clojure.lang.ExceptionInfo :cljs ExceptionInfo)+ #"control-boom"+ (experiment/experiment {:name :test/control-throws :min-interval-ms 0}+ (throw (ex-info "control-boom" {:from :control}))+ :candidate))))++(deftest candidate-side-effects-test+ (testing "candidate runs when experiment is enabled and not throttled"+ (let [side-effect (atom false)]+ (experiment/experiment {:name :test/side-effect :min-interval-ms 0}+ :control+ (do (reset! side-effect true) :candidate))+ (is (true? @side-effect))))+ (testing "candidate does NOT run when globally disabled"+ (let [side-effect (atom false)]+ (experiment/set-experiments-enabled-fn! (constantly false))+ (try+ (experiment/experiment {:name :test/disabled :min-interval-ms 0}+ :control+ (do (reset! side-effect true) :candidate))+ (is (false? @side-effect))+ (finally+ (experiment/set-experiments-enabled-fn! (constantly true)))))))++;;; ---------------------------------------------- Outcome comparison ----------------------------------------------++(deftest matching-results-test+ (let [[report-fn results] (capture-report-fn)]+ (experiment/experiment {:name :test/match :min-interval-ms 0 :report-fn report-fn}+ 42+ 42)+ (is (= 1 (count @results)))+ (is (true? (:match? (first @results))))))++(deftest mismatching-results-test+ (let [[report-fn results] (capture-report-fn)]+ (experiment/experiment {:name :test/mismatch :min-interval-ms 0 :report-fn report-fn}+ :old+ :new)+ (is (= 1 (count @results)))+ (is (false? (:match? (first @results))))))++(deftest candidate-throws-control-doesnt-test+ (let [[report-fn results] (capture-report-fn)]+ (experiment/experiment {:name :test/candidate-error :min-interval-ms 0 :report-fn report-fn}+ :control+ (throw (ex-info "boom" {})))+ (is (false? (:match? (first @results))))+ (is (contains? (:candidate-outcome (first @results)) :error))))++(deftest control-throws-candidate-doesnt-test+ (let [[report-fn results] (capture-report-fn)]+ (is (thrown?+ #?(:clj clojure.lang.ExceptionInfo :cljs ExceptionInfo)+ (experiment/experiment {:name :test/control-error :min-interval-ms 0 :report-fn report-fn}+ (throw (ex-info "control-boom" {}))+ :candidate)))+ (is (= 1 (count @results)))+ (is (false? (:match? (first @results))))))++(deftest both-throw-same-exception-test+ (let [[report-fn results] (capture-report-fn)]+ (is (thrown?+ #?(:clj clojure.lang.ExceptionInfo :cljs ExceptionInfo)+ (experiment/experiment {:name :test/both-throw-same :min-interval-ms 0 :report-fn report-fn}+ (throw (ex-info "boom" {:code 42}))+ (throw (ex-info "boom" {:code 42})))))+ (is (= 1 (count @results)))+ (is (true? (:match? (first @results))))))++(deftest both-throw-different-exception-test+ (let [[report-fn results] (capture-report-fn)]+ (is (thrown?+ #?(:clj clojure.lang.ExceptionInfo :cljs ExceptionInfo)+ (experiment/experiment {:name :test/both-throw-diff :min-interval-ms 0 :report-fn report-fn}+ (throw (ex-info "boom-a" {:code 1}))+ (throw (ex-info "boom-b" {:code 2})))))+ (is (= 1 (count @results)))+ (is (false? (:match? (first @results))))))++(deftest custom-comparator-test+ (let [[report-fn results] (capture-report-fn)+ close-enough? (fn [a b] (< (abs (- a b)) 0.01))]+ (experiment/experiment {:name :test/custom-cmp+ :min-interval-ms 0+ :report-fn report-fn+ :comparator-fn close-enough?}+ 1.0+ 1.005)+ (is (true? (:match? (first @results))))))++;;; --------------------------------------------------- Metrics ---------------------------------------------------++(deftest report-fn-receives-durations-test+ (let [[report-fn results] (capture-report-fn)]+ (experiment/experiment {:name :test/durations :min-interval-ms 0 :report-fn report-fn}+ #?(:clj (do (Thread/sleep 1) :control)+ :cljs :control)+ #?(:clj (do (Thread/sleep 1) :candidate)+ :cljs :candidate))+ (let [r (first @results)]+ (is (pos? (:control-duration-ns r)))+ (is (pos? (:candidate-duration-ns r))))))++(deftest report-fn-receives-name-test+ (let [[report-fn results] (capture-report-fn)]+ (experiment/experiment {:name :test/name-check :min-interval-ms 0 :report-fn report-fn}+ :a+ :a)+ (is (= :test/name-check (:name (first @results))))))++;;; -------------------------------------------------- Throttling -------------------------------------------------++(deftest throttle-no-limit-test+ (let [[report-fn results] (capture-report-fn)]+ (dotimes [_ 5]+ (experiment/experiment {:name :test/no-throttle :min-interval-ms 0 :report-fn report-fn}+ :control+ :candidate))+ (is (= 5 (count @results)))))++(deftest throttle-skips-rapid-calls-test+ (let [[report-fn results] (capture-report-fn)+ ;; unique per run so the throttle atom's leftover state from previous runs doesn't affect us+ name-key (keyword "test" (str "throttled-" (gensym)))]+ (experiment/experiment {:name name-key :min-interval-ms 60000 :report-fn report-fn}+ :control+ :candidate)+ (experiment/experiment {:name name-key :min-interval-ms 60000 :report-fn report-fn}+ :control+ :candidate)+ (is (= 1 (count @results)))))++;;; ------------------------------------------------- Kill switch -------------------------------------------------++(deftest global-kill-switch-test+ (testing "experiments-enabled false prevents candidate from running"+ (let [[report-fn results] (capture-report-fn)]+ (experiment/set-experiments-enabled-fn! (constantly false))+ (try+ (experiment/experiment {:name :test/killed :min-interval-ms 0 :report-fn report-fn}+ :control+ :candidate)+ (is (empty? @results))+ (finally+ (experiment/set-experiments-enabled-fn! (constantly true))))))+ (testing "re-enabling allows experiments again"+ (let [[report-fn results] (capture-report-fn)]+ (experiment/experiment {:name :test/re-enabled :min-interval-ms 0 :report-fn report-fn}+ :control+ :candidate)+ (is (= 1 (count @results))))))++;;; -------------------------------------------- Async execution (JVM) --------------------------------------------++#?(:clj+ (deftest async-execution-test+ (testing "candidate runs asynchronously by default on JVM"+ (binding [experiment/*sync?* false]+ (let [p (promise)]+ (experiment/experiment {:name :test/async+ :min-interval-ms 0+ :report-fn #(deliver p %)}+ :control+ :control)+ (let [result (deref p 5000 ::timeout)]+ (is (not= ::timeout result))+ (is (true? (:match? result)))))))))++;;; ------------------------------------------- Default report fn --------------------------------------------------++(defn- with-default-report-fn! [f body-fn]+ (let [prev @#?(:clj @#'experiment/default-report-fn+ :cljs experiment/default-report-fn)]+ (try+ (experiment/set-default-report-fn! f)+ (body-fn)+ (finally+ (experiment/set-default-report-fn! prev)))))++(deftest default-report-fn-test+ (testing "the default report fn is used when no :report-fn is specified"+ (let [[report-fn results] (capture-report-fn)]+ (with-default-report-fn!+ report-fn+ (fn []+ (experiment/experiment {:name :test/default-report :min-interval-ms 0}+ :a+ :a)))+ (is (= 1 (count @results)))+ (is (true? (:match? (first @results))))))+ (testing ":report-fn overrides the default report fn"+ (let [[default-report-fn default-results] (capture-report-fn)+ [custom-report-fn custom-results] (capture-report-fn)]+ (with-default-report-fn!+ default-report-fn+ (fn []+ (experiment/experiment {:name :test/override-report+ :min-interval-ms 0+ :report-fn custom-report-fn}+ :a+ :a)))+ (is (empty? @default-results))+ (is (= 1 (count @custom-results))))))