Skip to content

A peer-to-peer stack for code collaboration

Run Radicle tests from script

Thomas Scholtes92659130e010
diff --git a/cloudbuild.yaml b/cloudbuild.yaml
index bff3644..b54697d 100644
--- a/cloudbuild.yaml
+++ b/cloudbuild.yaml
@@ -48,6 +48,8 @@ steps:
apt-get -yqq install libpq-dev
stack build --fast --test --no-terminal --pedantic
+ stack exec -- radicle test/prelude.rad
+
./images/radicle-server/ci-copy-bin.sh
- id: "Lint (weeder)"
diff --git a/docs/source/guide/Testing.rst b/docs/source/guide/Testing.rst
new file mode 100644
index 0000000..70433ae
--- /dev/null
+++ b/docs/source/guide/Testing.rst
@@ -0,0 +1,75 @@
+Testing
+=======
+
+Radicle provides a ``:test`` macro which allows you to write tests next
+to your code.
+
+.. code-block:: radicle
+
+ (def not
+ (fn [x] (if x #f #t)))
+
+ (:test "not"
+ [ (not #t) ==> #f ]
+ [ (not #f) ==> #t ]
+ )
+
+The ``test/prelude.rad`` script runs all tests defined in the prelude.
+
+
+Test Definition
+---------------
+
+Each test definition consists of a test name and a list of steps
+
+.. code-block:: radicle
+
+ (:test "my test" step1 step2 ...)
+
+Each step is a vector triple with the symbol ``==>`` in the middle. For
+example
+
+.. code-block:: radicle
+
+ [ (not #t) ==> #f ]
+
+When a test step is run the value left of ``==>`` is evaluated in the
+environment captured at the definition site of the test. The resulting value is
+then compared with the right-hand side. The test passes if both are
+equal. The right-hand side is *not* evaluated.
+
+If the evaluation of the left-hand side throws an error the test fails
+with the message produced by the error.
+
+Changes to reference in a test step evaluation have no effect and all
+tests steps are run independently.
+
+
+Test Setup
+----------
+
+There is a special setup test step that allows you to change the
+environment that tests steps run in.
+
+.. code-block:: radicle
+
+ (:test "with setup"
+ [ :setup (do
+ (def foo 5)
+ )]
+ [ (+ foo 2) ==> 7 ]
+ [ (- foo 2) ==> 3 ]
+ )
+
+Similar to test steps the body of the ``:setup`` step is evaluated in
+the environment of the definition site. Changes to the environment
+introduced by evaluating the setup code are then available in all test
+steps.
+
+
+Running Tests
+-------------
+
+All tests defined with the ``:test`` macro are collected in the ``tests``
+reference. Tests can be executed using the ``run-all`` function from
+the ``prelude/test`` module.
diff --git a/docs/source/index.rst b/docs/source/index.rst
index bab6e5c..3dff018 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -14,6 +14,7 @@ radicle
guide/Basics.lrad
guide/Chains.lrad
guide/DataValidation.rst
+ guide/Testing.rst
guide/Issues.lrad
reference.rst
diff --git a/docs/source/reference.rst b/docs/source/reference.rst
index c3e70b7..362d2a8 100644
--- a/docs/source/reference.rst
+++ b/docs/source/reference.rst
@@ -1130,6 +1130,13 @@ last input in ``inputs``. The ``index`` argument is either ``:nothing``
in which case all inputs are fetched or ``[:just i]`` in which case all
inputs following after the index ``i`` are fetched.
+``(install-remote-chain-fake)``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Install test doubles for the ``send!`` and ``receive!`` primitives that
+use a mutable dictionary to store RSMs. Requires
+``rad/test/stub-primitives`` to be loaded
+
``prelude/state-machine``
-------------------------
diff --git a/rad/prelude/chain.rad b/rad/prelude/chain.rad
index 70de6ac..9e8c4dd 100644
--- a/rad/prelude/chain.rad
+++ b/rad/prelude/chain.rad
@@ -2,12 +2,14 @@
:doc "Functions for simulating remote chains."
:exports '[new-chain load-chain! eval-in-chain update-chain-ref! eval
updatable-eval update-chain! eval-fn-app send-prelude!
- send-code! send! receive!]}
+ send-code! send! receive! install-remote-chain-fake]}
(import prelude/io '[read-file-values!] :as 'io)
(import prelude/ref '[modify-ref] :unqualified)
-(import prelude/lens '[.. ... view over @] :unqualified)
+(import prelude/lens '[.. ... view over @ @def] :unqualified)
(import prelude/basic :unqualified)
+(import prelude/io :unqualified)
+(import prelude/patterns :unqualified)
;; Chains: Functions for dealing with chains.
@@ -64,12 +66,9 @@
{:chain new-chain
:result result }))
-(def local-test-chain-name
- (fn [] (string-append "http://localhost:8000/chains/" (uuid!))))
-
(:test "eval-in-chain"
[:setup
- (do (def empty-chain (new-chain (local-test-chain-name)))
+ (do (def empty-chain (new-chain (uuid!)))
(def res1 (eval-in-chain '(def x 3) empty-chain))
(def res (eval-in-chain '(+ x 2) (lookup :chain res1))))
]
@@ -94,10 +93,32 @@
(def chain (foldl upd-ch chain new-inputs))
(insert :index [:just new-index] chain)))
+(def install-remote-chain-fake
+ "Install test doubles for the `send!` and `receive!` primitives that
+ use a mutable dictionary to store RSMs. Requires
+ `rad/test/stub-primitives` to be loaded"
+ (fn []
+ (def remote-chains (ref {}))
+
+ (write-ref primitive-stub-ref/send!
+ (fn [name exprs]
+ (modify-ref remote-chains
+ (fn [chains] (over (@def name []) (fn [chain-exprs] (<> chain-exprs exprs)) chains)))))
+
+ (write-ref primitive-stub-ref/receive!
+ (fn [name maybeIndex]
+ (def inputs (lookup name (read-ref remote-chains)))
+ (def index
+ (match maybeIndex
+ (/just 'index) (+ index 1)
+ :nothing 0))
+ [(length inputs) (drop index inputs)]))
+ ))
(:test "update-chain!"
[:setup
(do
- (def chain-name (local-test-chain-name))
+ (install-remote-chain-fake)
+ (def chain-name (uuid!))
(def chain (new-chain chain-name))
(send! chain-name ['(+ 3 2)])
(send! chain-name ['(+ 3 3)])
@@ -122,7 +143,8 @@
(:test "update-chain-ref!"
[:setup
(do
- (def chain-name (local-test-chain-name))
+ (install-remote-chain-fake)
+ (def chain-name (uuid!))
(def chain (ref (new-chain chain-name)))
(send! chain-name ['(+ 3 2)])
(send! chain-name ['(+ 3 3)])
diff --git a/rad/prelude/lens.rad b/rad/prelude/lens.rad
index cf16189..86d21b4 100644
--- a/rad/prelude/lens.rad
+++ b/rad/prelude/lens.rad
@@ -90,7 +90,6 @@
==> {:a {:b {:c 1}}} ])
-
(def @def
"Returns a lens targetting keys of dicts with a default value for getting if
the key does not exist in the target."
diff --git a/rad/prelude/test.rad b/rad/prelude/test.rad
index 0481559..0846aa9 100644
--- a/rad/prelude/test.rad
+++ b/rad/prelude/test.rad
@@ -2,6 +2,12 @@
:doc "Provides eval that adds a `:test` macro."
:exports '[run run-all assert-equal]}
+
+(import prelude/basic :unqualified)
+(import prelude/bool :unqualified)
+(import prelude/list :unqualified)
+(import prelude/patterns :unqualified)
+
(def assert-equal
"Compares `actual` with `expected` with `eq?`. If the values are not equal an
`'assertion-error` is thrown. Otherwise `actual` is retuned."
@@ -11,6 +17,65 @@
(throw 'assertion-error (string-append "failed: got " (show actual) " but expected " (show expected))))
))
+;; Utils
+
+(def try
+ "Call `f` and catch any errors with `tag`. If no errors were thrown
+ then return `[:ok r]` where `r` is the result of calling `f`. If an
+ error was thrown return `[:error e]` where `e` is the second argument
+ to `throw`."
+ (fn [tag f]
+ (def handle-error
+ (fn [msg] [:error msg]))
+ (catch tag [:ok (f)] handle-error)
+ ))
+
+(def foldl-with-index
+ "Like `(fold f r xs)` but evaluates `(f acc x i)` where `i` is an
+ additional index argument that starts with 0."
+ (fn [f r xs]
+ (head
+ (foldl
+ (fn [acc x]
+ (match acc ['r 'i] [(f r x i) (+ i 1)]))
+ [r 0]
+ xs))
+ ))
+
+(def parse-doc-test-step
+ "Parse a doc test step of the form `[ actual ==> expected ]` and
+ returns a function that when called with an environments runs the
+ step.
+
+ Running a doc step means evluating `actual` in the given environment
+ and comparing the result with `expected` using `assert-equal`."
+ (fn [step]
+ (def lhs (nth 0 step))
+ (def marker (nth 1 step))
+ (def rhs (nth 2 step))
+ (if (eq? marker '==>)
+ (fn [env]
+ (def lhs-res (head (eval lhs env)))
+ (assert-equal lhs-res rhs))
+ (throw 'parse-test-step "Expected and actual not separate by ==>"))
+ ))
+
+(def extract-setup
+ "Extracts the a setup step from the test steps. Returns a `[setup steps]` tuple."
+ (fn [steps]
+ (match steps
+ (/cons [:setup 'setup] 'steps)
+ [setup steps]
+ 'steps
+ [:nothing steps])
+ ))
+
+(def print-test-result
+ "Print a test result to stdout using the Test Anyhting Protocol (TAP)."
+ (fn [name result i]
+ (match result
+ [:ok _] (put-str! (string-append "ok " (show i) " - " name))
+ [:error 'msg] (put-str! (string-append "not ok " (show i) " - " name)))))
(def run
"Run the tests in `test-def` and print the result. `test-def` is a `{:env env
@@ -26,36 +91,33 @@
)
```
"
- (fn [test-def]
+ (fn [test-def index]
(def test-env (lookup :env test-def))
(def test-name (lookup :name test-def))
- (def error
- (fn [msg]
- (put-str! (string-append "Test '" test-name "' failed: " msg))))
- ;; Check a single test in a given env
- (def check-single
- (fn [single ix senv]
- (def lhs (nth 0 single))
- (def marker (nth 1 single))
- (def rhs (nth 2 single))
- (if (eq? marker '==>)
- (do
- (def lhs-res (first (eval lhs senv)))
- (if (eq? lhs-res rhs)
- (put-str! (string-append "Test '" test-name " (" (show ix) ")' succeeded"))
- (error (string-append "failed: got " (show lhs-res) " but expected " (show rhs)))))
- (error "Expected and actual not separate by ==>"))
- (+ ix 1)))
(def test-contents (lookup :tests test-def))
- (def maybe-setup (nth 0 test-contents))
- ;; If first block starts with :setup, run tests in that env
- (if (eq? (nth 0 maybe-setup) :setup)
- (do (def setup-env (nth 1 (eval (nth 1 maybe-setup) test-env)))
- (foldl (fn [ix x] (check-single x ix setup-env)) 1 (rest test-contents)))
- (foldl (fn [ix x] (check-single x ix test-env)) 1 test-contents))
+ (def setup-and-steps (extract-setup (lookup :tests test-def)))
+ (def setup (nth 0 setup-and-steps))
+ (def steps (nth 1 setup-and-steps))
+ (def step-runners (map parse-doc-test-step steps))
+ (def test-env (nth 1 (eval setup test-env)))
+ (def result
+ (try 'any
+ (fn []
+ (map (fn [r] (r test-env)) step-runners)
+ :nil)))
+ (print-test-result test-name result index)
+ (match result
+ [:error _] #f
+ [:ok _] #t)
))
+
(def run-all
- "Run all `tests`."
+ "Run all `tests` and print the results. Returns `#f` if at least one test fails and `#t` otherwise."
(fn [tests]
- (map (fn [t] (run t)) tests)))
+ (put-str! (string-append "1.." (show (length tests))))
+ (foldl-with-index
+ (fn [ok t i] (and ok (run t (+ 1 i))))
+ #t
+ tests)
+ ))
diff --git a/rad/tests/stub-primitives.rad b/rad/tests/stub-primitives.rad
new file mode 100644
index 0000000..0719038
--- /dev/null
+++ b/rad/tests/stub-primitives.rad
@@ -0,0 +1,21 @@
+;; This script stubs some primitives for code evaluated afterwards.
+;;
+;; This code is used by `rad/tests/run.rad` to test code that relies on
+;; primitives. Currently only the `send!` and `receive!` primitives are stubbed.
+;;
+;; A primitive `prim` is stubbed by creating a ref `primitive-stub-ref/prim`
+;; that holds the original primitive value. The function `prim` is then
+;; redefined as calling the function stored in the reference.
+;;
+;; See `prelude/chain/install-remote-chain-fake` for how to use primitve stubs.
+
+;; send!
+(def primitive-stub-ref/send! (ref machine/eval-server/update!))
+(def machine/eval-server/update!
+ (fn [a b] ((read-ref primitive-stub-ref/send!) a b)))
+
+;; receive!
+(def primitive-stub-ref/receive! (ref machine/eval-server/get-log!))
+(def machine/eval-server/get-log!
+ (fn [a b] ((read-ref primitive-stub-ref/receive!) a b)))
+
diff --git a/test/prelude.rad b/test/prelude.rad
new file mode 100755
index 0000000..3f29344
--- /dev/null
+++ b/test/prelude.rad
@@ -0,0 +1,12 @@
+#!/usr/bin/env radicle
+
+;; Run all tests defined in the prelude and print their results with
+;; TAP. Exit with status code 1 if any of the tests fail.
+
+(load! "rad/tests/stub-primitives.rad")
+(load! "rad/prelude.rad")
+
+(import prelude/test)
+
+(def tests-ok (prelude/test/run-all (read-ref tests)))
+(if tests-ok (exit! 0) (exit! 1))
diff --git a/test/spec/Radicle/Tests.hs b/test/spec/Radicle/Tests.hs
index 1a6ce3b..fdf70b1 100644
--- a/test/spec/Radicle/Tests.hs
+++ b/test/spec/Radicle/Tests.hs
@@ -830,8 +830,7 @@ test_cbor =
-- Radicle source with the @:test@ macro.
test_source_files :: IO TestTree
test_source_files = do
- tests <- join <$> traverse testOne [ "rad/prelude.rad"
- , "rad/monadic/issues.rad"
+ tests <- join <$> traverse testOne [ "rad/monadic/issues.rad"
, "rad/monadic/names.rad"
]
pure $ testGroup "Radicle source file tests" tests
@@ -843,7 +842,8 @@ test_source_files = do
Right kp -> pure $ renderCompactPretty kp
Left _ -> panic "Couldn't generate keypair file."
let ws = addFileToWorld "my-keys.rad" keyPair ws'
- let code = "(load! \"" <> file <> "\")"
+ let code = "(load! \"rad/tests/stub-primitives.rad\")"
+ <> "(load! \"" <> file <> "\")"
<> "(import prelude/test)"
<> "(prelude/test/run-all (read-ref tests))"
(r, out) <- runCodeWithWorld ws $ toS code