Skip to content

A peer-to-peer stack for code collaboration

Simple data validation (#188)

James Haydonb7ac47f29cfd
diff --git a/docs/source/guide/DataValidation.rst b/docs/source/guide/DataValidation.rst
new file mode 100644
index 0000000..21a3cd5
--- /dev/null
+++ b/docs/source/guide/DataValidation.rst
@@ -0,0 +1,91 @@
+Data Validation
+==================
+
+Most of the entities that get manipulated and stored on chains are represented
+by structured radicle data. To maintain the integrety of the chain it can be
+important to refuse malformed/invalid data. To detect this, use *validator
+functions*. These are functions which take a piece of data as input and either
+throw an exception (thus cancelling the current transaction), or return the data
+unchanged. The radicle prelude contains functions for creating and combining
+validators.
+
+For example let's assume that we want to validate the ``:priority`` field on an
+issue entity, with the requirement that it be an integer :math:`p` such that
+:math:`0 \leq p \leq 10`. To do this we can define a validator:
+
+.. code-block:: radicle
+
+ (def validator/priority
+ (validator/and
+ [(validator/type :number)
+ (validator/pred "is integer"
+ integral?)
+ (validator/pred "0 <= p <= 10"
+ (fn [p] (and (< -1 p) (< p 11))))]))
+
+Here we have used ``validator/and`` to combine several validators together: the
+priority is valid only if it passes all three validators:
+
+- The first one checks that the value is a number.
+
+- The second uses ``validator/pred``, which takes a description and a predicate,
+and checks that the predicate holds. In this case we ensure that the priority is
+a whole number.
+
+- The third validator ensures the priority is in the required range.
+
+Very often entities are represented using dicts with specific required keys.
+Validators for these entities can be created using the ``validator/keys``
+function. For example to validate an issue we could use:
+
+.. code-block:: radicle
+
+ (def validator/issue
+ (validator/keys
+ {:id (validator/and
+ [(validator/type :string)
+ (validator/pred "valid UUID"
+ uuid?)])
+ :author (validator/pred "valid public key"
+ public-key?)
+ :title (validator/and
+ [(validator/type :string)
+ (validator/pred "< 60 chars"
+ (fn [s] (< (string-length s) 60)))])
+ :body (validator/and
+ [(validator/type :string)
+ (validator/pred "valid markdown"
+ markdown?)
+ (validator/pred "< 4000 chars"
+ (fn [s] (< (string-length s) 4000)))])
+ :tags (validator/every
+ (validator/and
+ [(validator/type :string)
+ (validator/pred "< 40 chars"
+ (fn [s] (< (string-length s) 40)))]))
+ :priority validator/priority}))
+
+This checks that an issue is a dict, with required keys:
+
+- ``:id`` that's a valid UUID string,
+
+- ``:author`` that's a valid public key,
+
+- ``:title`` that's a string shorter than 60 characters,
+
+- ``:body`` that's a markdown string less that 4000 characters,
+
+- ``:tag`` that's a vector of strings, each shorted than 40 characters.
+
+- ``:priority`` that's a integers :math:`p` such that :math:`0 \leq p \leq 10`.
+
+Because validators return their input if it is valid, we can just wrap any value
+which should be validated with the appropriate validator:
+
+.. code-block:: radicle
+
+ (fn [issue]
+ (modify-ref
+ issues
+ (fn [is]
+ (add-right (validator/issue issue) is))))
diff --git a/docs/source/guide/index.rst b/docs/source/guide/index.rst
index 9f8fb39..afcab95 100644
--- a/docs/source/guide/index.rst
+++ b/docs/source/guide/index.rst
@@ -59,3 +59,4 @@ how to allow for future changes to the chain.
GettingStarted.md
Basics.lrad
Chains.lrad
+ DataValidation.rst
diff --git a/docs/source/index.rst b/docs/source/index.rst
index 54d368f..bab6e5c 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -13,6 +13,7 @@ radicle
guide/GettingStarted.rst
guide/Basics.lrad
guide/Chains.lrad
+ guide/DataValidation.rst
guide/Issues.lrad
reference.rst
diff --git a/exe/ReferenceDoc.hs b/exe/ReferenceDoc.hs
index e4c2681..4fac624 100644
--- a/exe/ReferenceDoc.hs
+++ b/exe/ReferenceDoc.hs
@@ -16,8 +16,7 @@ import Radicle.Internal.Doc (md)
import System.Console.Haskeline (defaultSettings, runInputT)
import Text.Pandoc
-main ::
- IO ()
+main :: IO ()
main = do
res_ <- runInputT defaultSettings $
interpret
@@ -62,6 +61,10 @@ main = do
"Effectful functions. These functions are not available in 'pure' chains, but are available in the local REPL."
, sec "Lenses" lens
"Functional references into radicle values."
+ , sec "Validation" validation
+ [md|Functions for creating or combining *validators*, which are functions which return the
+ input unchanged or throw with an error message. These can be used for checking data before
+ accepting it onto a chain.|]
, sec "Cryptography" crypto
"Tools for creating and verifying cryptographic signatures, and generating private/public key pairs."
, sec "Chain tools" chainTools
@@ -71,13 +74,14 @@ main = do
]
basics =
- [ "eq?", "not", "and", "or", "all", "some", "show", "string-append", "apply", "type", "atom?"
- , "boolean?", "string?", "number?", "keyword?", "list?", "dict?", "read", "throw"
- , "Y", "Y2", "to-json", "uuid?", "make-counter" ]
- maths = ["+", "*", "-", "<", ">"]
+ [ "eq?", "not", "and", "or", "all", "some", "show", "string-append", "string-length"
+ , "apply", "type", "atom?", "boolean?", "string?", "number?", "keyword?", "list?"
+ , "dict?", "read", "throw", "Y", "Y2", "to-json", "uuid?", "make-counter", "markdown?"
+ , "public-key?" ]
+ maths = ["+", "*", "-", "<", ">", "integral?"]
evalFns = ["base-eval", "eval", "updatable-eval"]
envStuff = ["pure-env", "get-current-env", "set-current-env", "set-env!"]
- seqs = ["nth", "foldl", "foldr", "map", "seq"]
+ seqs = ["nth", "foldl", "foldr", "map", "drop", "seq"]
lists =
[ "list", "nil", "head", "tail", "empty?", "cons", "reverse", "length", "concat"
, "filter", "range", "list-with-head" ]
@@ -92,6 +96,9 @@ main = do
["print!", "get-line!", "load!", "read-file!", "read-code!", "send-code!"
, "send-prelude!", "subscribe-to!", "uuid!", "read-line!", "exit!"]
lens = ["@", "make-lens", "view", "view-ref", "set", "set-ref", "over", "over-ref", "id-lens", "..", "..."]
+ validation =
+ [ "validator/=", "validator/member", "validator/type", "validator/pred", "validator/every"
+ , "validator/and", "validator/or", "validator/key", "validator/keys"]
crypto = ["verify-signature", "default-ecc-curve", "gen-key-pair!", "gen-signature!"]
chainTools =
[ "new-chain", "eval-in-chain", "enter-remote-chain", "update-chain", "add-quit", "add-send"
@@ -105,7 +112,7 @@ main = do
allFns =
basics ++ maths ++ evalFns ++ envStuff ++ seqs ++ lists ++ vecs
- ++ dicts ++ structs ++ refs ++ docs ++ io ++ lens ++ crypto ++ chainTools
+ ++ dicts ++ structs ++ refs ++ docs ++ io ++ lens ++ validation ++ crypto ++ chainTools
++ doNotInclude
-- Function symbol followed by a hard line break, followed by it's doc.
@@ -128,7 +135,7 @@ main = do
then pure ()
else panic $ "The following functions need to be added to the reference doc: " <> T.intercalate ", " notDocumented
- lPanic (Left _) m = panic m
+ lPanic (Left e) m = panic $ m <> ": " <> show e
lPanic (Right r) _ = pure r
inlinePandoc t = case runPure (readMarkdown Default.def t) of
diff --git a/rad/prelude.rad b/rad/prelude.rad
index 70894d8..0cd58ab 100644
--- a/rad/prelude.rad
+++ b/rad/prelude.rad
@@ -9,6 +9,7 @@
(load! "rad/prelude/exception.rad")
(load! "rad/chain.rad")
(load! "rad/state-machine.rad")
+(load! "rad/prelude/validation.rad")
(def pure-prelude-files
(list
@@ -21,6 +22,7 @@
"rad/prelude/lens.rad"
"rad/prelude/exception.rad"
"rad/state-machine.rad"
+ "rad/prelude/validation.rad"
))
(document 'pure-prelude-files
diff --git a/rad/prelude/validation.rad b/rad/prelude/validation.rad
new file mode 100644
index 0000000..8d9e61c
--- /dev/null
+++ b/rad/prelude/validation.rad
@@ -0,0 +1,81 @@
+(def validator/=
+ "Given `x`, returns a validator that checks for equality with `x`."
+ (fn [x]
+ (fn [y]
+ (if (eq? x y)
+ x
+ (throw 'validation-failure (string-append "Not equal to: " (show x)))))))
+
+(def validator/member
+ "Given a structure, returns a validator which checks for membership in the structure."
+ (fn [xs]
+ (fn [x]
+ (if (member? x xs)
+ x
+ (throw 'validation-failure (string-append "Not a member of: " (show xs)))))))
+
+(def validator/and
+ "Given a sequence of validators `vs`, returns a new validator which, given a
+ value, checks if it conforms to all the validators in `vs`."
+ (fn [vs]
+ (fn [x]
+ (map (fn [v] (v x)) vs)
+ x)))
+
+(def-rec validator/or
+ "Given a vector of validators `vs`, returns a new validator which, given a
+ value, checks if it conforms to at least one of the `vs`."
+ (fn [vs]
+ (fn [x]
+ (if (eq? vs [])
+ (throw 'validation-failure "validation/or: didn't satisfy any of the validators")
+ (catch
+ 'validation-failure
+ ((nth 0 vs) x)
+ (fn [y] ((validator/or (drop 1 vs)) x)))))))
+
+(def validator/type
+ "Checks that a value has a type. Expects a keyword describing the type, as
+ returned by the `type` function."
+ (fn [t]
+ (fn [x]
+ (if (eq? (type x) t)
+ x
+ (throw 'validation-failure (string-append "Did not have type " (show t) ": " (show x)))))))
+
+(def validator/pred
+ "Given a description and a predicate, returns a validator that checks if the
+ predicate is true."
+ (fn [name p]
+ (fn [x]
+ (if (p x)
+ x
+ (throw 'validation-failure (string-append "Did not satisfy predicate: " name))))))
+
+(def validator/key
+ "Given a key and a validator, returns a validator which checks for
+ the existence of that key and that the associated value conforms to the validator."
+ (fn [k v]
+ (fn [d]
+ (if (member? k d)
+ (v (lookup k d))
+ (throw 'validation-failure (string-append "Missing required key: " (show k)))))))
+
+(def validator/keys
+ "Given a dict associating keys to validators, returns a validator which checks a dict
+ for the existence of those keys, and that they conform to the associated validators."
+ (fn [ks]
+ (fn [d]
+ (if (dict? d)
+ ((validator/and
+ (map (fn [kv] (validator/key (nth 0 kv) (nth 1 kv)))
+ (seq ks)))
+ d)
+ (throw 'validation-failure (string-append "Not a dict: " (show d)))))))
+
+(def validator/every
+ "Given a validator, creates a new validator which checks that all the items in
+ a sequence conform to it."
+ (fn [v]
+ (fn [xs]
+ (map v xs))))
diff --git a/scripts/build-docs.sh b/scripts/build-docs.sh
index 9982696..aeb8689 100755
--- a/scripts/build-docs.sh
+++ b/scripts/build-docs.sh
@@ -24,7 +24,7 @@ popd () {
}
build(){
- stack exec ref-doc
+ stack exec radicle-ref-doc
pushd "$PWD"/docs
[ -z "$IS_NIX" ] && pip install -r requirements.txt
make html
diff --git a/src/Radicle/Internal/Core.hs b/src/Radicle/Internal/Core.hs
index 8bc1bed..838d371 100644
--- a/src/Radicle/Internal/Core.hs
+++ b/src/Radicle/Internal/Core.hs
@@ -441,27 +441,15 @@ specialForms = Map.fromList $ first Ident <$>
[v] -> pure v
xs -> throwErrorHere $ WrongNumberOfArgs "quote" 1 (length xs))
, ("def", \case
- [Atom name, val] -> do
- val' <- baseEval val
- defineAtom name Nothing val'
- pure nil
+ [Atom name, val] -> def name Nothing val
[_, _] -> throwErrorHere $ OtherError "def expects atom for first arg"
- [Atom name, String d, val] -> do
- val' <- baseEval val
- defineAtom name (Just d) val'
- pure nil
+ [Atom name, String d, val] -> def name (Just d) val
xs -> throwErrorHere $ WrongNumberOfArgs "def" 2 (length xs))
, ( "def-rec"
, \case
- [Atom name, val] -> do
- val' <- baseEval val
- case val' of
- Lambda is b e -> do
- let v = Lambda is b (Env . Map.insert name (Doc.Docd Nothing v) . fromEnv $ e)
- defineAtom name Nothing v
- pure nil
- _ -> throwErrorHere $ OtherError "def-rec can only be used to define functions"
+ [Atom name, val] -> defRec name Nothing val
[_, _] -> throwErrorHere $ OtherError "def-rec expects atom for first arg"
+ [Atom name, String d, val] -> defRec name (Just d) val
xs -> throwErrorHere $ WrongNumberOfArgs "def-rec" 2 (length xs)
)
, ("do", (lastDef nil <$>) . traverse baseEval)
@@ -496,6 +484,20 @@ specialForms = Map.fromList $ first Ident <$>
then baseEval e
else cond ps
+ def name doc_ val = do
+ val' <- baseEval val
+ defineAtom name doc_ val'
+ pure nil
+
+ defRec name doc_ val = do
+ val' <- baseEval val
+ case val' of
+ Lambda is b e -> do
+ let v = Lambda is b (Env . Map.insert name (Doc.Docd doc_ v) . fromEnv $ e)
+ defineAtom name doc_ v
+ pure nil
+ _ -> throwErrorHere $ OtherError "def-rec can only be used to define functions"
+
-- * From/ToRadicle
type CPA t = (Ann.Annotation t, Copointed t)
diff --git a/src/Radicle/Internal/PrimFns.hs b/src/Radicle/Internal/PrimFns.hs
index 38a8622..f93f1cf 100644
--- a/src/Radicle/Internal/PrimFns.hs
+++ b/src/Radicle/Internal/PrimFns.hs
@@ -5,12 +5,15 @@ module Radicle.Internal.PrimFns where
import Protolude hiding (TypeError)
import qualified Data.Aeson as Aeson
+import qualified Data.Default as Default
import qualified Data.IntMap as IntMap
import qualified Data.Map as Map
import Data.Scientific (Scientific, floatingOrInteger)
import Data.Sequence (Seq(..))
import qualified Data.Sequence as Seq
+import qualified Data.Text as T
import GHC.Exts (IsList(..))
+import qualified Text.Pandoc as Pandoc
import qualified Radicle.Internal.Annotation as Ann
import Radicle.Internal.Core
@@ -166,6 +169,18 @@ purePrimFns = fromList $ allDocs $
_ -> throwErrorHere $ TypeError "tail: expects list argument")
-- Lists and Vecs
+ , ( "drop"
+ , [md|Returns all but the first `n` items of a sequence, unless the sequence is empty,
+ in which case an exception is thrown.|]
+ , twoArg "drop" $ \case
+ (Number n, vs) -> case floatingOrInteger n of
+ Left (_ :: Double) -> throwErrorHere $ OtherError "drop: first argument must be an integer"
+ Right i -> case vs of
+ List xs -> pure . List $ drop i xs
+ Vec xs -> pure . Vec $ Seq.drop i xs
+ _ -> throwErrorHere $ TypeError $ "drop: second argument must be a list of vector"
+ _ -> throwErrorHere $ TypeError $ "drop: first argument must be a number"
+ )
, ( "nth"
, [md|Given an integral number `n` and `xs`, returns the `n`th element
(zero indexed) of `xs` when `xs` is a list or a vector. If `xs`
@@ -205,7 +220,13 @@ purePrimFns = fromList $ allDocs $
pure (Dict (Map.fromList (zip (fst <$> kvs) vs)))
_ -> throwErrorHere $ TypeError $ "map-values: second argument must be a dict"
)
- , ("string-append"
+ , ( "string-length"
+ , [md|Returns the length of a string.|]
+ , oneArg "string-length" $ \case
+ String s -> pure . Number . fromIntegral . T.length $ s
+ _ -> throwErrorHere $ TypeError "string-length: expecting string"
+ )
+ , ( "string-append"
, [md|Concatenates a variable number of string arguments. If one of the arguments
isn't a string then an exception is thrown.|]
, \args ->
@@ -214,7 +235,15 @@ purePrimFns = fromList $ allDocs $
ss = fromStr <$> args
in if all isJust ss
then pure . String . mconcat $ catMaybes ss
- else throwErrorHere $ TypeError "string-append: non-string argument")
+ else throwErrorHere $ TypeError "string-append: non-string argument"
+ )
+ , ( "markdown?"
+ , [md|Checks that the input string is valid Markdown according
+ to the [CommonMark spec](commonmark.org).|]
+ , oneArg "markdown?" $ \case
+ String s -> pure . Boolean . isRight $ Pandoc.runPure (Pandoc.readCommonMark Default.def s)
+ _ -> throwErrorHere $ TypeError "markdown?: expects string"
+ )
, ( "insert"
, [md|Given `k`, `v` and a dict `d`, returns a dict with the same associations
as `d` but with `k` associated to `d`. If `d` isn't a dict then an exception
@@ -255,6 +284,14 @@ purePrimFns = fromList $ allDocs $
[Number x, Number y] -> pure $ Boolean (x > y)
[_, _] -> throwErrorHere $ TypeError ">: expecting number"
xs -> throwErrorHere $ WrongNumberOfArgs ">" 2 (length xs))
+ , ( "integral?"
+ , [md|Checks if a number is an integer.|]
+ , oneArg "integral?" $ \case
+ Number n -> case floatingOrInteger n of
+ Left (_ :: Double) -> pure $ Boolean False
+ Right (_ :: Integer) -> pure $ Boolean True
+ _ -> throwErrorHere $ TypeError "integral?: expecting number"
+ )
, ( "foldl"
, [md|Given a function `f`, an initial value `i` and a sequence (list or vector)
`xs`, reduces `xs` to a single value by starting with `i` and repetitively
@@ -370,14 +407,14 @@ purePrimFns = fromList $ allDocs $
, oneArg "show" (pure . String . renderPrettyDef))
, ( "seq"
, [md|Given a structure `s`, returns a sequence. Lists and vectors are returned
- without modification while for dicts a list of key-value-pairs is returned:
- these are lists of length 2 whose first item is a key and whose second item
+ without modification while for dicts a vector of key-value-pairs is returned:
+ these are vectors of length 2 whose first item is a key and whose second item
is the associated value.|]
, oneArg "seq" $
\case
x@(List _) -> pure x
x@(Vec _) -> pure x
- Dict kvs -> pure $ List [List [k, v] | (k,v) <- Map.toList kvs ]
+ Dict kvs -> pure . Vec . Seq.fromList $ [Vec (Seq.fromList [k, v]) | (k,v) <- Map.toList kvs ]
_ -> throwErrorHere $ TypeError "seq: can only be used on a list, vector or dictionary"
)
, ( "to-json"
@@ -403,6 +440,13 @@ purePrimFns = fromList $ allDocs $
[_, _, _] -> throwErrorHere $ OtherError "verify-signature: message must be a string"
xs -> throwErrorHere $ WrongNumberOfArgs "verify-signature" 3 (length xs)
)
+ , ( "public-key?"
+ , [md|Checks if a value represents a valid public key.|]
+ , oneArg "public-key?" $
+ \v -> case fromRad v of
+ Right (_ :: PublicKey) -> pure $ Boolean True
+ Left _ -> pure $ Boolean False
+ )
, ( "uuid?"
, [md|Checks if a string has the format of a UUID.|]
, oneArg "uuid?" $ \case
diff --git a/test/spec/Radicle/Tests.hs b/test/spec/Radicle/Tests.hs
index 23cd9ae..a5cb56a 100644
--- a/test/spec/Radicle/Tests.hs
+++ b/test/spec/Radicle/Tests.hs
@@ -71,6 +71,10 @@ test_eval =
let prog = [s|(tail (list #t #f #t))|]
prog `succeedsWith` List [Boolean False, Boolean True]
+ , testCase "'drop' drops" $ do
+ "(drop 1 (list #t #f #t))" `succeedsWith` List [Boolean False, Boolean True]
+ "(drop 2 [1 2 3 4])" `succeedsWith` toRad [3 :: Int, 4]
+
, testCase "'nth' extracts elements from lists and vectors" $ do
"(nth 0 [0 1 2])" `succeedsWith` Number 0
"(nth 1 [0 1 2])" `succeedsWith` Number 1
@@ -96,6 +100,11 @@ test_eval =
-- Either evaluation failed or their equal.
counterexample prog $ isLeft res || res == Right (Boolean False)
+ , testCase "'integral?' returns true for whole numbers and false fractional numbers" $ do
+ "(integral? 0)" `succeedsWith` Boolean True
+ "(integral? 1)" `succeedsWith` Boolean True
+ "(integral? 3.14159265359)" `succeedsWith` Boolean False
+
, testCase "'member?' returns true if list contains element" $ do
let prog = [s|(member? #t (list #f #t))|]
prog `succeedsWith` Boolean True
@@ -137,6 +146,14 @@ test_eval =
"Got:\n" <> prettyEither res
counterexample (toS info) $ res == expected
+ , testProperty "'string-length' returns length of a string" $ \x -> do
+ let prog = "(string-length " <> renderPrettyDef (asValue (String x)) <> ")"
+ res = runTest' prog
+ expected = Right $ Number $ fromIntegral $ T.length x
+ info = "Expected:\n" <> prettyEither (Right (String x)) <>
+ "Got:\n" <> prettyEither (runTest' prog)
+ counterexample (toS info) $ res == expected
+
, testCase "'foldl' foldls the list" $ do
let prog = [s|(foldl (fn [x y] (- x y)) 0 (list 1 2 3))|]
prog `succeedsWith` Number (-6)