Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Introduce experiments (#72794)

metamben190b23d30fe7
diff --git a/src/metabase/analytics/experiment.cljc b/src/metabase/analytics/experiment.cljc
new file mode 100644
index 00000000000..42ee5c6bc2b
--- /dev/null
+++ b/src/metabase/analytics/experiment.cljc
@@ -0,0 +1,35 @@
+(ns metabase.analytics.experiment
+ "Default experiment report function backed by metabase.analytics-interface.core.
+ Requiring this namespace wires `report!` as the default report fn."
+ (:require
+ [metabase.analytics-interface.core :as analytics]
+ [metabase.util.experiment :as experiment]))
+
+(defn report!
+ "Report an experiment result via the analytics interface (Prometheus on JVM, batched POST on CLJS).
+
+ Expects a map with keys:
+ :name - keyword identifying the experiment
+ :match? - whether control and candidate results matched
+ :control-duration-ns - nanoseconds the control took
+ :candidate-duration-ns - nanoseconds the candidate took
+ :control-outcome - {:result v} or {:error t}
+ :candidate-outcome - {:result v} or {:error t}"
+ [{:keys [name match? control-duration-ns candidate-duration-ns candidate-outcome]}]
+ (let [labels {:experiment (cond-> (str name)
+ (keyword? name) (subs 1))}
+ error? (contains? candidate-outcome :error)
+ outcome-key (cond
+ match? :experiment/matches-total
+ error? :experiment/errors-total
+ :else :experiment/mismatches-total)
+ candidate-duration-key (if error?
+ :experiment/candidate-error-duration-ms
+ :experiment/candidate-duration-ms)]
+ (analytics/inc! :experiment/runs-total labels)
+ (analytics/inc! outcome-key labels)
+ (analytics/inc! :experiment/control-duration-ms labels (/ control-duration-ns 1e6))
+ (analytics/inc! candidate-duration-key labels (/ candidate-duration-ns 1e6))))
+
+;; Wire as the default report fn. Requiring this namespace is sufficient.
+(experiment/set-default-report-fn! report!)
diff --git a/src/metabase/analytics/prometheus.clj b/src/metabase/analytics/prometheus.clj
index 86655cba059..671368f5a89 100644
--- a/src/metabase/analytics/prometheus.clj
+++ b/src/metabase/analytics/prometheus.clj
@@ -646,7 +646,30 @@
{:description "Number of frontend analytics events dropped before being POSTed to the backend, because the client-side buffer was full."})
(prometheus/counter :metabase-export/errors
{:description "Number of errors during data export."
- :labels [:format]})])
+ :labels [:format]})
+
+ ;; experiment metrics
+ (prometheus/counter :experiment/runs-total
+ {:description "Number of experiment candidate runs."
+ :labels [:experiment]})
+ (prometheus/counter :experiment/matches-total
+ {:description "Number of experiment runs where control and candidate matched."
+ :labels [:experiment]})
+ (prometheus/counter :experiment/mismatches-total
+ {:description "Number of experiment runs where control and candidate differed."
+ :labels [:experiment]})
+ (prometheus/counter :experiment/errors-total
+ {:description "Number of experiment runs where the candidate threw."
+ :labels [:experiment]})
+ (prometheus/counter :experiment/control-duration-ms
+ {:description "Cumulative duration in milliseconds of experiment control code path."
+ :labels [:experiment]})
+ (prometheus/counter :experiment/candidate-duration-ms
+ {:description "Cumulative duration in milliseconds of experiment candidate code path."
+ :labels [:experiment]})
+ (prometheus/counter :experiment/candidate-error-duration-ms
+ {:description "Cumulative duration in milliseconds of experiment candidate code path when it threw."
+ :labels [:experiment]})])
(defn- quartz-collectors
[]
diff --git a/src/metabase/lib/js.cljs b/src/metabase/lib/js.cljs
index 2d99322c848..572c11f832c 100644
--- a/src/metabase/lib/js.cljs
+++ b/src/metabase/lib/js.cljs
@@ -61,6 +61,7 @@
[goog.object :as gobject]
[medley.core :as m]
[metabase.analytics-interface.core :as analytics.interface]
+ [metabase.analytics.experiment]
[metabase.analytics.impl]
^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.normalize :as mbql.normalize]
[metabase.lib.aggregation :as lib.aggregation]
@@ -98,8 +99,10 @@
;;; This ensures that all of metabase.lib.* is loaded, so all the `defmethod`s are properly registered.
;;; metabase.analytics.impl registers the CLJS reporter for [[metabase.analytics-interface.core]].
+;;; metabase.analytics.experiment wires the default experiment report fn.
(comment lib.core/keep-me
+ metabase.analytics.experiment/keep-me
metabase.analytics.impl/keep-me)
;; Expose for E2E testing
diff --git a/src/metabase/util/experiment.cljc b/src/metabase/util/experiment.cljc
new file mode 100644
index 00000000000..76f6a1cd7f0
--- /dev/null
+++ b/src/metabase/util/experiment.cljc
@@ -0,0 +1,173 @@
+(ns metabase.util.experiment
+ "Framework for running experimental code paths alongside
+ production code, comparing results, and reporting metrics.
+
+ Usage:
+ (experiment {:name :my-experiment :min-interval-ms 5000}
+ (production-code)
+ (experimental-code))
+
+ Always returns the result of the production (control) code path.
+ The candidate is only run when the global kill switch is on and
+ the time-based throttle allows it."
+ (:require
+ [metabase.util.log :as log]
+ [net.cgrand.macrovich :as macros])
+ #?(:cljs (:require-macros [metabase.util.experiment])))
+
+;;; ------------------------------------------------ Configuration ------------------------------------------------
+
+(defonce ^{:private true
+ :doc "Atom holding a zero-arg function that returns whether experiments are enabled.
+ Defaults to (constantly true). On JVM, wired to the setting getter at startup.
+ On CLJS, wired to read from bootstrap data."}
+ experiments-enabled-fn
+ (atom (constantly true)))
+
+(defn experiments-enabled?
+ "Returns true if experiments are globally enabled. Checks the setting on every call."
+ []
+ (@experiments-enabled-fn))
+
+(defn set-experiments-enabled-fn!
+ "Set the function that determines whether experiments are enabled.
+ Called once at application startup to wire in the setting."
+ [f]
+ (reset! experiments-enabled-fn f))
+
+(defonce ^{:private true
+ :doc "Atom holding the default report function. Set at application startup via [[set-default-report-fn!]].
+ Nil means no reporting. Can be overridden per-experiment via :report-fn in the opts map."}
+ default-report-fn (atom nil))
+
+(defn set-default-report-fn!
+ "Set the default report function for experiment results."
+ [f]
+ (reset! default-report-fn f))
+
+(def ^:dynamic *sync?*
+ "When true, forces synchronous candidate execution on JVM. For testing only."
+ false)
+
+;;; -------------------------------------------------- Throttling -------------------------------------------------
+
+(defonce ^:private last-run-times (atom {}))
+
+(defn- current-time-ms []
+ #?(:clj (System/currentTimeMillis)
+ :cljs (js/Date.now)))
+
+(defn- throttle-allows?
+ "Returns true if enough time has elapsed since the last run of this experiment."
+ [experiment-name min-interval-ms]
+ (let [now (current-time-ms)
+ last-run (get @last-run-times experiment-name 0)]
+ (when (>= (- now last-run) min-interval-ms)
+ (swap! last-run-times assoc experiment-name now)
+ true)))
+
+;;; --------------------------------------------- Outcome comparison ----------------------------------------------
+
+(defn- outcomes-match?
+ "Compare control and candidate outcomes. Both are {:result v} or {:error t}.
+ Results match via comparator-fn. Errors match if same type, ex-message, and ex-data."
+ [comparator-fn control candidate]
+ (try
+ (cond
+ (and (contains? control :result) (contains? candidate :result))
+ (comparator-fn (:result control) (:result candidate))
+
+ (and (contains? control :error) (contains? candidate :error))
+ (let [ce (:error control)
+ ca (:error candidate)]
+ (and (= (type ce) (type ca))
+ (= (ex-message ce) (ex-message ca))
+ (= (ex-data ce) (ex-data ca))))
+
+ :else false)
+ (catch #?(:clj Throwable :cljs :default) _
+ false)))
+
+;;; ------------------------------------------- Candidate execution -----------------------------------------------
+
+(defn nanotime
+ "Returns current time in nanoseconds. Cross-platform. Public because the `experiment` macro
+ expands calls to this in the caller's namespace."
+ []
+ #?(:clj (System/nanoTime)
+ :cljs (* 1000000 (js/performance.now))))
+
+(defn run-experiment*
+ "Runtime implementation. Called by the `experiment` macro. Do not call directly.
+ Checks the global kill switch and throttle, and if both allow, runs the candidate."
+ [{:keys [name min-interval-ms comparator-fn report-fn]} control-outcome control-duration-ns candidate-thunk]
+ (when (and (experiments-enabled?)
+ (throttle-allows? name (or min-interval-ms 1000)))
+ (let [comparator-fn (or comparator-fn =)
+ report-fn (or report-fn @default-report-fn)
+ run-fn
+ (fn []
+ (let [start (nanotime)
+ candidate-outcome (try
+ {:result (candidate-thunk)}
+ (catch #?(:clj Throwable :cljs :default) t
+ {:error t}))
+ end (nanotime)
+ duration (- end start)
+ match? (outcomes-match? comparator-fn control-outcome candidate-outcome)]
+ (when-not match?
+ (if (contains? candidate-outcome :error)
+ (log/warnf "Experiment %s: candidate threw %s" name (str (:error candidate-outcome)))
+ (log/warnf "Experiment %s: result MISMATCH" name)))
+ (when report-fn
+ (report-fn {:name name
+ :match? match?
+ :control-duration-ns control-duration-ns
+ :candidate-duration-ns duration
+ :control-outcome control-outcome
+ :candidate-outcome candidate-outcome}))))]
+ ;; NOTE: `future-call` conveys thread bindings automatically via `binding-conveyor-fn`.
+ ;; If switching to a custom executor, capture bindings with `(get-thread-bindings)` before
+ ;; submission, then restore them with `(push-thread-bindings)` / `(pop-thread-bindings)`
+ ;; around the body in the worker thread.
+ #?(:clj (if *sync?* (run-fn) (future-call run-fn))
+ :cljs (run-fn)))))
+
+;;; --------------------------------------------------- Macro ----------------------------------------------------
+
+(defmacro experiment
+ "Run an experiment comparing control and candidate code paths.
+
+ Always returns the control result (or re-throws the control exception).
+ The candidate is only run when the global kill switch is on and
+ the time-based throttle allows it.
+
+ Options map:
+ :name - keyword identifying this experiment (required)
+ :min-interval-ms - minimum ms between candidate runs (default 1000)
+ :comparator-fn - 2-arg fn for comparing results (default =)
+ :report-fn - 1-arg fn called with result map (default: the fn set via set-default-report-fn!)
+
+ Usage:
+ (experiment {:name :my-experiment}
+ (control-expression)
+ (candidate-expression))"
+ {:style/indent 1}
+ [opts control candidate]
+ (assert (:name opts) ":name is required for experiment")
+ `(let [start# (nanotime)
+ control-outcome# (try
+ {:result ~control}
+ (catch ~(macros/case :clj 'Throwable :cljs ':default) t#
+ {:error t#}))
+ duration# (- (nanotime) start#)]
+ (run-experiment*
+ ~opts
+ control-outcome#
+ duration#
+ ~(macros/case
+ :clj `(^:once fn* [] ~candidate)
+ :cljs `(fn [] ~candidate)))
+ (if (contains? control-outcome# :error)
+ (throw (:error control-outcome#))
+ (:result control-outcome#))))
diff --git a/src/metabase/util/experiment/settings.clj b/src/metabase/util/experiment/settings.clj
new file mode 100644
index 00000000000..2b9b4c3f416
--- /dev/null
+++ b/src/metabase/util/experiment/settings.clj
@@ -0,0 +1,18 @@
+(ns metabase.util.experiment.settings
+ (:require
+ [metabase.settings.core :refer [defsetting]]
+ [metabase.util.experiment :as experiment]
+ [metabase.util.i18n :refer [deferred-tru]]))
+
+#_{:clj-kondo/ignore [:metabase/defsetting-namespace]}
+(defsetting experiments-enabled
+ (deferred-tru "Enable or disable all code experiments. When disabled, only the production code path runs.")
+ :type :boolean
+ :default false
+ :visibility :admin
+ :export? false
+ :audit :getter)
+
+;; Wire the setting into the experiment system. The function is called on every experiment
+;; invocation, so toggling the setting in the admin panel takes effect immediately.
+(experiment/set-experiments-enabled-fn! experiments-enabled)
diff --git a/test/metabase/analytics/experiment_test.cljc b/test/metabase/analytics/experiment_test.cljc
new file mode 100644
index 00000000000..80ca4906cf2
--- /dev/null
+++ b/test/metabase/analytics/experiment_test.cljc
@@ -0,0 +1,120 @@
+(ns metabase.analytics.experiment-test
+ (:require
+ [clojure.test :refer [deftest is testing]]
+ [metabase.analytics-interface.core :as analytics.interface]
+ [metabase.analytics.experiment :as analytics.experiment]
+ [metabase.util.experiment :as experiment]))
+
+(defn- make-test-reporter [calls]
+ (reify analytics.interface/Reporter
+ (-inc! [_ metric labels amount]
+ (swap! calls conj {:op :inc! :metric metric :labels labels :amount amount}))
+ (-dec-gauge! [_ metric labels amount]
+ (swap! calls conj {:op :dec-gauge! :metric metric :labels labels :amount amount}))
+ (-set-gauge! [_ metric labels amount]
+ (swap! calls conj {:op :set-gauge! :metric metric :labels labels :amount amount}))
+ (-observe! [_ metric labels amount]
+ (swap! calls conj {:op :observe! :metric metric :labels labels :amount amount}))
+ (-clear! [_ metric]
+ (swap! calls conj {:op :clear! :metric metric}))))
+
+(defn- do-with-test-reporter! [f]
+ (let [calls (atom [])
+ original-reporter (analytics.interface/get-reporter)]
+ (try
+ (analytics.interface/set-reporter! (make-test-reporter calls))
+ (f calls)
+ (finally
+ (analytics.interface/set-reporter! original-reporter)))))
+
+(defn- calls-for [calls metric]
+ (filterv #(= metric (:metric %)) @calls))
+
+;;; ---------------------------------------------------- Tests ----------------------------------------------------
+
+(deftest report-match-test
+ (testing "matching result increments runs-total and matches-total, observes durations"
+ (do-with-test-reporter!
+ (fn [calls]
+ (analytics.experiment/report!
+ {:name :test/match
+ :match? true
+ :control-duration-ns 1000000
+ :candidate-duration-ns 2000000
+ :control-outcome {:result 42}
+ :candidate-outcome {:result 42}})
+ (let [labels {:experiment "test/match"}]
+ (is (= [{:op :inc! :metric :experiment/runs-total :labels labels :amount 1}]
+ (calls-for calls :experiment/runs-total)))
+ (is (= [{:op :inc! :metric :experiment/matches-total :labels labels :amount 1}]
+ (calls-for calls :experiment/matches-total)))
+ (is (empty? (calls-for calls :experiment/mismatches-total)))
+ (is (empty? (calls-for calls :experiment/errors-total)))
+ (is (= [{:op :inc! :metric :experiment/control-duration-ms :labels labels :amount 1.0}]
+ (calls-for calls :experiment/control-duration-ms)))
+ (is (= [{:op :inc! :metric :experiment/candidate-duration-ms :labels labels :amount 2.0}]
+ (calls-for calls :experiment/candidate-duration-ms))))))))
+
+(deftest report-mismatch-test
+ (testing "mismatching result increments runs-total and mismatches-total"
+ (do-with-test-reporter!
+ (fn [calls]
+ (analytics.experiment/report!
+ {:name :test/mismatch
+ :match? false
+ :control-duration-ns 1000000
+ :candidate-duration-ns 2000000
+ :control-outcome {:result :old}
+ :candidate-outcome {:result :new}})
+ (let [labels {:experiment "test/mismatch"}]
+ (is (= [{:op :inc! :metric :experiment/runs-total :labels labels :amount 1}]
+ (calls-for calls :experiment/runs-total)))
+ (is (empty? (calls-for calls :experiment/matches-total)))
+ (is (= [{:op :inc! :metric :experiment/mismatches-total :labels labels :amount 1}]
+ (calls-for calls :experiment/mismatches-total)))
+ (is (empty? (calls-for calls :experiment/errors-total))))))))
+
+(deftest report-matching-errors-test
+ (testing "both threw same exception: counted as match, not error"
+ (do-with-test-reporter!
+ (fn [calls]
+ (let [ex (ex-info "boom" {:code 1})]
+ (analytics.experiment/report!
+ {:name :test/matching-error
+ :match? true
+ :control-duration-ns 1000000
+ :candidate-duration-ns 800000
+ :control-outcome {:error ex}
+ :candidate-outcome {:error ex}}))
+ (let [labels {:experiment "test/matching-error"}]
+ (is (= [{:op :inc! :metric :experiment/matches-total :labels labels :amount 1}]
+ (calls-for calls :experiment/matches-total)))
+ (is (empty? (calls-for calls :experiment/errors-total)))
+ (is (empty? (calls-for calls :experiment/mismatches-total))))))))
+
+(deftest report-candidate-error-test
+ (testing "candidate error (mismatch) increments runs-total and errors-total"
+ (do-with-test-reporter!
+ (fn [calls]
+ (analytics.experiment/report!
+ {:name :test/error
+ :match? false
+ :control-duration-ns 1000000
+ :candidate-duration-ns 500000
+ :control-outcome {:result :ok}
+ :candidate-outcome {:error (ex-info "boom" {})}})
+ (let [labels {:experiment "test/error"}]
+ (is (= [{:op :inc! :metric :experiment/runs-total :labels labels :amount 1}]
+ (calls-for calls :experiment/runs-total)))
+ (is (= [{:op :inc! :metric :experiment/errors-total :labels labels :amount 1}]
+ (calls-for calls :experiment/errors-total)))
+ (is (empty? (calls-for calls :experiment/matches-total)))
+ (is (empty? (calls-for calls :experiment/mismatches-total)))
+ (is (empty? (calls-for calls :experiment/candidate-duration-ms)))
+ (is (= [{:op :inc! :metric :experiment/candidate-error-duration-ms :labels labels :amount 0.5}]
+ (calls-for calls :experiment/candidate-error-duration-ms))))))))
+
+(deftest wiring-test
+ (testing "requiring metabase.analytics.experiment installs report! as the default report fn"
+ (is (= analytics.experiment/report!
+ @@#'experiment/default-report-fn))))
diff --git a/test/metabase/util/experiment_test.cljc b/test/metabase/util/experiment_test.cljc
new file mode 100644
index 00000000000..c3cf1bfa32d
--- /dev/null
+++ b/test/metabase/util/experiment_test.cljc
@@ -0,0 +1,243 @@
+(ns metabase.util.experiment-test
+ (:require
+ [clojure.test :refer [deftest is testing use-fixtures]]
+ [metabase.util.experiment :as experiment])
+ #?(:cljs (:require-macros [metabase.util.experiment])))
+
+(defn- capture-report-fn
+ "Returns [report-fn, results-atom]. The report-fn appends each result map to the atom."
+ []
+ (let [results (atom [])]
+ [(fn [result] (swap! results conj result))
+ results]))
+
+(use-fixtures :each
+ (fn [t]
+ (binding [experiment/*sync?* true]
+ (t))))
+
+;;; ------------------------------------------------ Core behavior ------------------------------------------------
+
+(deftest control-result-returned-test
+ (testing "when candidate matches"
+ (is (= :control
+ (experiment/experiment {:name :test/match :min-interval-ms 0}
+ :control
+ :control))))
+ (testing "when candidate mismatches"
+ (is (= :control
+ (experiment/experiment {:name :test/mismatch :min-interval-ms 0}
+ :control
+ :candidate))))
+ (testing "when candidate throws"
+ (is (= :control
+ (experiment/experiment {:name :test/candidate-throws :min-interval-ms 0}
+ :control
+ (throw (ex-info "boom" {})))))))
+
+(deftest control-exception-rethrown-test
+ (is (thrown-with-msg?
+ #?(:clj clojure.lang.ExceptionInfo :cljs ExceptionInfo)
+ #"control-boom"
+ (experiment/experiment {:name :test/control-throws :min-interval-ms 0}
+ (throw (ex-info "control-boom" {:from :control}))
+ :candidate))))
+
+(deftest candidate-side-effects-test
+ (testing "candidate runs when experiment is enabled and not throttled"
+ (let [side-effect (atom false)]
+ (experiment/experiment {:name :test/side-effect :min-interval-ms 0}
+ :control
+ (do (reset! side-effect true) :candidate))
+ (is (true? @side-effect))))
+ (testing "candidate does NOT run when globally disabled"
+ (let [side-effect (atom false)]
+ (experiment/set-experiments-enabled-fn! (constantly false))
+ (try
+ (experiment/experiment {:name :test/disabled :min-interval-ms 0}
+ :control
+ (do (reset! side-effect true) :candidate))
+ (is (false? @side-effect))
+ (finally
+ (experiment/set-experiments-enabled-fn! (constantly true)))))))
+
+;;; ---------------------------------------------- Outcome comparison ----------------------------------------------
+
+(deftest matching-results-test
+ (let [[report-fn results] (capture-report-fn)]
+ (experiment/experiment {:name :test/match :min-interval-ms 0 :report-fn report-fn}
+ 42
+ 42)
+ (is (= 1 (count @results)))
+ (is (true? (:match? (first @results))))))
+
+(deftest mismatching-results-test
+ (let [[report-fn results] (capture-report-fn)]
+ (experiment/experiment {:name :test/mismatch :min-interval-ms 0 :report-fn report-fn}
+ :old
+ :new)
+ (is (= 1 (count @results)))
+ (is (false? (:match? (first @results))))))
+
+(deftest candidate-throws-control-doesnt-test
+ (let [[report-fn results] (capture-report-fn)]
+ (experiment/experiment {:name :test/candidate-error :min-interval-ms 0 :report-fn report-fn}
+ :control
+ (throw (ex-info "boom" {})))
+ (is (false? (:match? (first @results))))
+ (is (contains? (:candidate-outcome (first @results)) :error))))
+
+(deftest control-throws-candidate-doesnt-test
+ (let [[report-fn results] (capture-report-fn)]
+ (is (thrown?
+ #?(:clj clojure.lang.ExceptionInfo :cljs ExceptionInfo)
+ (experiment/experiment {:name :test/control-error :min-interval-ms 0 :report-fn report-fn}
+ (throw (ex-info "control-boom" {}))
+ :candidate)))
+ (is (= 1 (count @results)))
+ (is (false? (:match? (first @results))))))
+
+(deftest both-throw-same-exception-test
+ (let [[report-fn results] (capture-report-fn)]
+ (is (thrown?
+ #?(:clj clojure.lang.ExceptionInfo :cljs ExceptionInfo)
+ (experiment/experiment {:name :test/both-throw-same :min-interval-ms 0 :report-fn report-fn}
+ (throw (ex-info "boom" {:code 42}))
+ (throw (ex-info "boom" {:code 42})))))
+ (is (= 1 (count @results)))
+ (is (true? (:match? (first @results))))))
+
+(deftest both-throw-different-exception-test
+ (let [[report-fn results] (capture-report-fn)]
+ (is (thrown?
+ #?(:clj clojure.lang.ExceptionInfo :cljs ExceptionInfo)
+ (experiment/experiment {:name :test/both-throw-diff :min-interval-ms 0 :report-fn report-fn}
+ (throw (ex-info "boom-a" {:code 1}))
+ (throw (ex-info "boom-b" {:code 2})))))
+ (is (= 1 (count @results)))
+ (is (false? (:match? (first @results))))))
+
+(deftest custom-comparator-test
+ (let [[report-fn results] (capture-report-fn)
+ close-enough? (fn [a b] (< (abs (- a b)) 0.01))]
+ (experiment/experiment {:name :test/custom-cmp
+ :min-interval-ms 0
+ :report-fn report-fn
+ :comparator-fn close-enough?}
+ 1.0
+ 1.005)
+ (is (true? (:match? (first @results))))))
+
+;;; --------------------------------------------------- Metrics ---------------------------------------------------
+
+(deftest report-fn-receives-durations-test
+ (let [[report-fn results] (capture-report-fn)]
+ (experiment/experiment {:name :test/durations :min-interval-ms 0 :report-fn report-fn}
+ #?(:clj (do (Thread/sleep 1) :control)
+ :cljs :control)
+ #?(:clj (do (Thread/sleep 1) :candidate)
+ :cljs :candidate))
+ (let [r (first @results)]
+ (is (pos? (:control-duration-ns r)))
+ (is (pos? (:candidate-duration-ns r))))))
+
+(deftest report-fn-receives-name-test
+ (let [[report-fn results] (capture-report-fn)]
+ (experiment/experiment {:name :test/name-check :min-interval-ms 0 :report-fn report-fn}
+ :a
+ :a)
+ (is (= :test/name-check (:name (first @results))))))
+
+;;; -------------------------------------------------- Throttling -------------------------------------------------
+
+(deftest throttle-no-limit-test
+ (let [[report-fn results] (capture-report-fn)]
+ (dotimes [_ 5]
+ (experiment/experiment {:name :test/no-throttle :min-interval-ms 0 :report-fn report-fn}
+ :control
+ :candidate))
+ (is (= 5 (count @results)))))
+
+(deftest throttle-skips-rapid-calls-test
+ (let [[report-fn results] (capture-report-fn)
+ ;; unique per run so the throttle atom's leftover state from previous runs doesn't affect us
+ name-key (keyword "test" (str "throttled-" (gensym)))]
+ (experiment/experiment {:name name-key :min-interval-ms 60000 :report-fn report-fn}
+ :control
+ :candidate)
+ (experiment/experiment {:name name-key :min-interval-ms 60000 :report-fn report-fn}
+ :control
+ :candidate)
+ (is (= 1 (count @results)))))
+
+;;; ------------------------------------------------- Kill switch -------------------------------------------------
+
+(deftest global-kill-switch-test
+ (testing "experiments-enabled false prevents candidate from running"
+ (let [[report-fn results] (capture-report-fn)]
+ (experiment/set-experiments-enabled-fn! (constantly false))
+ (try
+ (experiment/experiment {:name :test/killed :min-interval-ms 0 :report-fn report-fn}
+ :control
+ :candidate)
+ (is (empty? @results))
+ (finally
+ (experiment/set-experiments-enabled-fn! (constantly true))))))
+ (testing "re-enabling allows experiments again"
+ (let [[report-fn results] (capture-report-fn)]
+ (experiment/experiment {:name :test/re-enabled :min-interval-ms 0 :report-fn report-fn}
+ :control
+ :candidate)
+ (is (= 1 (count @results))))))
+
+;;; -------------------------------------------- Async execution (JVM) --------------------------------------------
+
+#?(:clj
+ (deftest async-execution-test
+ (testing "candidate runs asynchronously by default on JVM"
+ (binding [experiment/*sync?* false]
+ (let [p (promise)]
+ (experiment/experiment {:name :test/async
+ :min-interval-ms 0
+ :report-fn #(deliver p %)}
+ :control
+ :control)
+ (let [result (deref p 5000 ::timeout)]
+ (is (not= ::timeout result))
+ (is (true? (:match? result)))))))))
+
+;;; ------------------------------------------- Default report fn --------------------------------------------------
+
+(defn- with-default-report-fn! [f body-fn]
+ (let [prev @#?(:clj @#'experiment/default-report-fn
+ :cljs experiment/default-report-fn)]
+ (try
+ (experiment/set-default-report-fn! f)
+ (body-fn)
+ (finally
+ (experiment/set-default-report-fn! prev)))))
+
+(deftest default-report-fn-test
+ (testing "the default report fn is used when no :report-fn is specified"
+ (let [[report-fn results] (capture-report-fn)]
+ (with-default-report-fn!
+ report-fn
+ (fn []
+ (experiment/experiment {:name :test/default-report :min-interval-ms 0}
+ :a
+ :a)))
+ (is (= 1 (count @results)))
+ (is (true? (:match? (first @results))))))
+ (testing ":report-fn overrides the default report fn"
+ (let [[default-report-fn default-results] (capture-report-fn)
+ [custom-report-fn custom-results] (capture-report-fn)]
+ (with-default-report-fn!
+ default-report-fn
+ (fn []
+ (experiment/experiment {:name :test/override-report
+ :min-interval-ms 0
+ :report-fn custom-report-fn}
+ :a
+ :a)))
+ (is (empty? @default-results))
+ (is (= 1 (count @custom-results))))))