A peer-to-peer stack for code collaboration
Simple data validation (#188)
James Haydonb7ac47f29cfd
diff --git a/docs/source/guide/DataValidation.rst b/docs/source/guide/DataValidation.rstnew file mode 100644index 0000000..21a3cd5--- /dev/null+++ b/docs/source/guide/DataValidation.rst@@ -0,0 +1,91 @@+Data Validation+==================++Most of the entities that get manipulated and stored on chains are represented+by structured radicle data. To maintain the integrety of the chain it can be+important to refuse malformed/invalid data. To detect this, use *validator+functions*. These are functions which take a piece of data as input and either+throw an exception (thus cancelling the current transaction), or return the data+unchanged. The radicle prelude contains functions for creating and combining+validators.++For example let's assume that we want to validate the ``:priority`` field on an+issue entity, with the requirement that it be an integer :math:`p` such that+:math:`0 \leq p \leq 10`. To do this we can define a validator:++.. code-block:: radicle++ (def validator/priority+ (validator/and+ [(validator/type :number)+ (validator/pred "is integer"+ integral?)+ (validator/pred "0 <= p <= 10"+ (fn [p] (and (< -1 p) (< p 11))))]))++Here we have used ``validator/and`` to combine several validators together: the+priority is valid only if it passes all three validators:++- The first one checks that the value is a number.++- The second uses ``validator/pred``, which takes a description and a predicate,+and checks that the predicate holds. In this case we ensure that the priority is+a whole number.++- The third validator ensures the priority is in the required range.++Very often entities are represented using dicts with specific required keys.+Validators for these entities can be created using the ``validator/keys``+function. For example to validate an issue we could use:++.. code-block:: radicle++ (def validator/issue+ (validator/keys+ {:id (validator/and+ [(validator/type :string)+ (validator/pred "valid UUID"+ uuid?)])+ :author (validator/pred "valid public key"+ public-key?)+ :title (validator/and+ [(validator/type :string)+ (validator/pred "< 60 chars"+ (fn [s] (< (string-length s) 60)))])+ :body (validator/and+ [(validator/type :string)+ (validator/pred "valid markdown"+ markdown?)+ (validator/pred "< 4000 chars"+ (fn [s] (< (string-length s) 4000)))])+ :tags (validator/every+ (validator/and+ [(validator/type :string)+ (validator/pred "< 40 chars"+ (fn [s] (< (string-length s) 40)))]))+ :priority validator/priority}))++This checks that an issue is a dict, with required keys:++- ``:id`` that's a valid UUID string,++- ``:author`` that's a valid public key,++- ``:title`` that's a string shorter than 60 characters,++- ``:body`` that's a markdown string less that 4000 characters,++- ``:tag`` that's a vector of strings, each shorted than 40 characters.++- ``:priority`` that's a integers :math:`p` such that :math:`0 \leq p \leq 10`.++Because validators return their input if it is valid, we can just wrap any value+which should be validated with the appropriate validator:++.. code-block:: radicle++ (fn [issue]+ (modify-ref+ issues+ (fn [is]+ (add-right (validator/issue issue) is))))diff --git a/docs/source/guide/index.rst b/docs/source/guide/index.rstindex 9f8fb39..afcab95 100644--- a/docs/source/guide/index.rst+++ b/docs/source/guide/index.rst@@ -59,3 +59,4 @@ how to allow for future changes to the chain.GettingStarted.mdBasics.lradChains.lrad+ DataValidation.rstdiff --git a/docs/source/index.rst b/docs/source/index.rstindex 54d368f..bab6e5c 100644--- a/docs/source/index.rst+++ b/docs/source/index.rst@@ -13,6 +13,7 @@ radicleguide/GettingStarted.rstguide/Basics.lradguide/Chains.lrad+ guide/DataValidation.rstguide/Issues.lradreference.rstdiff --git a/exe/ReferenceDoc.hs b/exe/ReferenceDoc.hsindex e4c2681..4fac624 100644--- a/exe/ReferenceDoc.hs+++ b/exe/ReferenceDoc.hs@@ -16,8 +16,7 @@ import Radicle.Internal.Doc (md)import System.Console.Haskeline (defaultSettings, runInputT)import Text.Pandoc-main ::- IO ()+main :: IO ()main = dores_ <- runInputT defaultSettings $interpret@@ -62,6 +61,10 @@ main = do"Effectful functions. These functions are not available in 'pure' chains, but are available in the local REPL.", sec "Lenses" lens"Functional references into radicle values."+ , sec "Validation" validation+ [md|Functions for creating or combining *validators*, which are functions which return the+ input unchanged or throw with an error message. These can be used for checking data before+ accepting it onto a chain.|], sec "Cryptography" crypto"Tools for creating and verifying cryptographic signatures, and generating private/public key pairs.", sec "Chain tools" chainTools@@ -71,13 +74,14 @@ main = do]basics =- [ "eq?", "not", "and", "or", "all", "some", "show", "string-append", "apply", "type", "atom?"- , "boolean?", "string?", "number?", "keyword?", "list?", "dict?", "read", "throw"- , "Y", "Y2", "to-json", "uuid?", "make-counter" ]- maths = ["+", "*", "-", "<", ">"]+ [ "eq?", "not", "and", "or", "all", "some", "show", "string-append", "string-length"+ , "apply", "type", "atom?", "boolean?", "string?", "number?", "keyword?", "list?"+ , "dict?", "read", "throw", "Y", "Y2", "to-json", "uuid?", "make-counter", "markdown?"+ , "public-key?" ]+ maths = ["+", "*", "-", "<", ">", "integral?"]evalFns = ["base-eval", "eval", "updatable-eval"]envStuff = ["pure-env", "get-current-env", "set-current-env", "set-env!"]- seqs = ["nth", "foldl", "foldr", "map", "seq"]+ seqs = ["nth", "foldl", "foldr", "map", "drop", "seq"]lists =[ "list", "nil", "head", "tail", "empty?", "cons", "reverse", "length", "concat", "filter", "range", "list-with-head" ]@@ -92,6 +96,9 @@ main = do["print!", "get-line!", "load!", "read-file!", "read-code!", "send-code!", "send-prelude!", "subscribe-to!", "uuid!", "read-line!", "exit!"]lens = ["@", "make-lens", "view", "view-ref", "set", "set-ref", "over", "over-ref", "id-lens", "..", "..."]+ validation =+ [ "validator/=", "validator/member", "validator/type", "validator/pred", "validator/every"+ , "validator/and", "validator/or", "validator/key", "validator/keys"]crypto = ["verify-signature", "default-ecc-curve", "gen-key-pair!", "gen-signature!"]chainTools =[ "new-chain", "eval-in-chain", "enter-remote-chain", "update-chain", "add-quit", "add-send"@@ -105,7 +112,7 @@ main = doallFns =basics ++ maths ++ evalFns ++ envStuff ++ seqs ++ lists ++ vecs- ++ dicts ++ structs ++ refs ++ docs ++ io ++ lens ++ crypto ++ chainTools+ ++ dicts ++ structs ++ refs ++ docs ++ io ++ lens ++ validation ++ crypto ++ chainTools++ doNotInclude-- Function symbol followed by a hard line break, followed by it's doc.@@ -128,7 +135,7 @@ main = dothen pure ()else panic $ "The following functions need to be added to the reference doc: " <> T.intercalate ", " notDocumented- lPanic (Left _) m = panic m+ lPanic (Left e) m = panic $ m <> ": " <> show elPanic (Right r) _ = pure rinlinePandoc t = case runPure (readMarkdown Default.def t) ofdiff --git a/rad/prelude.rad b/rad/prelude.radindex 70894d8..0cd58ab 100644--- a/rad/prelude.rad+++ b/rad/prelude.rad@@ -9,6 +9,7 @@(load! "rad/prelude/exception.rad")(load! "rad/chain.rad")(load! "rad/state-machine.rad")+(load! "rad/prelude/validation.rad")(def pure-prelude-files(list@@ -21,6 +22,7 @@"rad/prelude/lens.rad""rad/prelude/exception.rad""rad/state-machine.rad"+ "rad/prelude/validation.rad"))(document 'pure-prelude-filesdiff --git a/rad/prelude/validation.rad b/rad/prelude/validation.radnew file mode 100644index 0000000..8d9e61c--- /dev/null+++ b/rad/prelude/validation.rad@@ -0,0 +1,81 @@+(def validator/=+ "Given `x`, returns a validator that checks for equality with `x`."+ (fn [x]+ (fn [y]+ (if (eq? x y)+ x+ (throw 'validation-failure (string-append "Not equal to: " (show x)))))))++(def validator/member+ "Given a structure, returns a validator which checks for membership in the structure."+ (fn [xs]+ (fn [x]+ (if (member? x xs)+ x+ (throw 'validation-failure (string-append "Not a member of: " (show xs)))))))++(def validator/and+ "Given a sequence of validators `vs`, returns a new validator which, given a+ value, checks if it conforms to all the validators in `vs`."+ (fn [vs]+ (fn [x]+ (map (fn [v] (v x)) vs)+ x)))++(def-rec validator/or+ "Given a vector of validators `vs`, returns a new validator which, given a+ value, checks if it conforms to at least one of the `vs`."+ (fn [vs]+ (fn [x]+ (if (eq? vs [])+ (throw 'validation-failure "validation/or: didn't satisfy any of the validators")+ (catch+ 'validation-failure+ ((nth 0 vs) x)+ (fn [y] ((validator/or (drop 1 vs)) x)))))))++(def validator/type+ "Checks that a value has a type. Expects a keyword describing the type, as+ returned by the `type` function."+ (fn [t]+ (fn [x]+ (if (eq? (type x) t)+ x+ (throw 'validation-failure (string-append "Did not have type " (show t) ": " (show x)))))))++(def validator/pred+ "Given a description and a predicate, returns a validator that checks if the+ predicate is true."+ (fn [name p]+ (fn [x]+ (if (p x)+ x+ (throw 'validation-failure (string-append "Did not satisfy predicate: " name))))))++(def validator/key+ "Given a key and a validator, returns a validator which checks for+ the existence of that key and that the associated value conforms to the validator."+ (fn [k v]+ (fn [d]+ (if (member? k d)+ (v (lookup k d))+ (throw 'validation-failure (string-append "Missing required key: " (show k)))))))++(def validator/keys+ "Given a dict associating keys to validators, returns a validator which checks a dict+ for the existence of those keys, and that they conform to the associated validators."+ (fn [ks]+ (fn [d]+ (if (dict? d)+ ((validator/and+ (map (fn [kv] (validator/key (nth 0 kv) (nth 1 kv)))+ (seq ks)))+ d)+ (throw 'validation-failure (string-append "Not a dict: " (show d)))))))++(def validator/every+ "Given a validator, creates a new validator which checks that all the items in+ a sequence conform to it."+ (fn [v]+ (fn [xs]+ (map v xs))))diff --git a/scripts/build-docs.sh b/scripts/build-docs.shindex 9982696..aeb8689 100755--- a/scripts/build-docs.sh+++ b/scripts/build-docs.sh@@ -24,7 +24,7 @@ popd () {}build(){- stack exec ref-doc+ stack exec radicle-ref-docpushd "$PWD"/docs[ -z "$IS_NIX" ] && pip install -r requirements.txtmake htmldiff --git a/src/Radicle/Internal/Core.hs b/src/Radicle/Internal/Core.hsindex 8bc1bed..838d371 100644--- a/src/Radicle/Internal/Core.hs+++ b/src/Radicle/Internal/Core.hs@@ -441,27 +441,15 @@ specialForms = Map.fromList $ first Ident <$>[v] -> pure vxs -> throwErrorHere $ WrongNumberOfArgs "quote" 1 (length xs)), ("def", \case- [Atom name, val] -> do- val' <- baseEval val- defineAtom name Nothing val'- pure nil+ [Atom name, val] -> def name Nothing val[_, _] -> throwErrorHere $ OtherError "def expects atom for first arg"- [Atom name, String d, val] -> do- val' <- baseEval val- defineAtom name (Just d) val'- pure nil+ [Atom name, String d, val] -> def name (Just d) valxs -> throwErrorHere $ WrongNumberOfArgs "def" 2 (length xs)), ( "def-rec", \case- [Atom name, val] -> do- val' <- baseEval val- case val' of- Lambda is b e -> do- let v = Lambda is b (Env . Map.insert name (Doc.Docd Nothing v) . fromEnv $ e)- defineAtom name Nothing v- pure nil- _ -> throwErrorHere $ OtherError "def-rec can only be used to define functions"+ [Atom name, val] -> defRec name Nothing val[_, _] -> throwErrorHere $ OtherError "def-rec expects atom for first arg"+ [Atom name, String d, val] -> defRec name (Just d) valxs -> throwErrorHere $ WrongNumberOfArgs "def-rec" 2 (length xs)), ("do", (lastDef nil <$>) . traverse baseEval)@@ -496,6 +484,20 @@ specialForms = Map.fromList $ first Ident <$>then baseEval eelse cond ps+ def name doc_ val = do+ val' <- baseEval val+ defineAtom name doc_ val'+ pure nil++ defRec name doc_ val = do+ val' <- baseEval val+ case val' of+ Lambda is b e -> do+ let v = Lambda is b (Env . Map.insert name (Doc.Docd doc_ v) . fromEnv $ e)+ defineAtom name doc_ v+ pure nil+ _ -> throwErrorHere $ OtherError "def-rec can only be used to define functions"+-- * From/ToRadicletype CPA t = (Ann.Annotation t, Copointed t)diff --git a/src/Radicle/Internal/PrimFns.hs b/src/Radicle/Internal/PrimFns.hsindex 38a8622..f93f1cf 100644--- a/src/Radicle/Internal/PrimFns.hs+++ b/src/Radicle/Internal/PrimFns.hs@@ -5,12 +5,15 @@ module Radicle.Internal.PrimFns whereimport Protolude hiding (TypeError)import qualified Data.Aeson as Aeson+import qualified Data.Default as Defaultimport qualified Data.IntMap as IntMapimport qualified Data.Map as Mapimport Data.Scientific (Scientific, floatingOrInteger)import Data.Sequence (Seq(..))import qualified Data.Sequence as Seq+import qualified Data.Text as Timport GHC.Exts (IsList(..))+import qualified Text.Pandoc as Pandocimport qualified Radicle.Internal.Annotation as Annimport Radicle.Internal.Core@@ -166,6 +169,18 @@ purePrimFns = fromList $ allDocs $_ -> throwErrorHere $ TypeError "tail: expects list argument")-- Lists and Vecs+ , ( "drop"+ , [md|Returns all but the first `n` items of a sequence, unless the sequence is empty,+ in which case an exception is thrown.|]+ , twoArg "drop" $ \case+ (Number n, vs) -> case floatingOrInteger n of+ Left (_ :: Double) -> throwErrorHere $ OtherError "drop: first argument must be an integer"+ Right i -> case vs of+ List xs -> pure . List $ drop i xs+ Vec xs -> pure . Vec $ Seq.drop i xs+ _ -> throwErrorHere $ TypeError $ "drop: second argument must be a list of vector"+ _ -> throwErrorHere $ TypeError $ "drop: first argument must be a number"+ ), ( "nth", [md|Given an integral number `n` and `xs`, returns the `n`th element(zero indexed) of `xs` when `xs` is a list or a vector. If `xs`@@ -205,7 +220,13 @@ purePrimFns = fromList $ allDocs $pure (Dict (Map.fromList (zip (fst <$> kvs) vs)))_ -> throwErrorHere $ TypeError $ "map-values: second argument must be a dict")- , ("string-append"+ , ( "string-length"+ , [md|Returns the length of a string.|]+ , oneArg "string-length" $ \case+ String s -> pure . Number . fromIntegral . T.length $ s+ _ -> throwErrorHere $ TypeError "string-length: expecting string"+ )+ , ( "string-append", [md|Concatenates a variable number of string arguments. If one of the argumentsisn't a string then an exception is thrown.|], \args ->@@ -214,7 +235,15 @@ purePrimFns = fromList $ allDocs $ss = fromStr <$> argsin if all isJust ssthen pure . String . mconcat $ catMaybes ss- else throwErrorHere $ TypeError "string-append: non-string argument")+ else throwErrorHere $ TypeError "string-append: non-string argument"+ )+ , ( "markdown?"+ , [md|Checks that the input string is valid Markdown according+ to the [CommonMark spec](commonmark.org).|]+ , oneArg "markdown?" $ \case+ String s -> pure . Boolean . isRight $ Pandoc.runPure (Pandoc.readCommonMark Default.def s)+ _ -> throwErrorHere $ TypeError "markdown?: expects string"+ ), ( "insert", [md|Given `k`, `v` and a dict `d`, returns a dict with the same associationsas `d` but with `k` associated to `d`. If `d` isn't a dict then an exception@@ -255,6 +284,14 @@ purePrimFns = fromList $ allDocs $[Number x, Number y] -> pure $ Boolean (x > y)[_, _] -> throwErrorHere $ TypeError ">: expecting number"xs -> throwErrorHere $ WrongNumberOfArgs ">" 2 (length xs))+ , ( "integral?"+ , [md|Checks if a number is an integer.|]+ , oneArg "integral?" $ \case+ Number n -> case floatingOrInteger n of+ Left (_ :: Double) -> pure $ Boolean False+ Right (_ :: Integer) -> pure $ Boolean True+ _ -> throwErrorHere $ TypeError "integral?: expecting number"+ ), ( "foldl", [md|Given a function `f`, an initial value `i` and a sequence (list or vector)`xs`, reduces `xs` to a single value by starting with `i` and repetitively@@ -370,14 +407,14 @@ purePrimFns = fromList $ allDocs $, oneArg "show" (pure . String . renderPrettyDef)), ( "seq", [md|Given a structure `s`, returns a sequence. Lists and vectors are returned- without modification while for dicts a list of key-value-pairs is returned:- these are lists of length 2 whose first item is a key and whose second item+ without modification while for dicts a vector of key-value-pairs is returned:+ these are vectors of length 2 whose first item is a key and whose second itemis the associated value.|], oneArg "seq" $\casex@(List _) -> pure xx@(Vec _) -> pure x- Dict kvs -> pure $ List [List [k, v] | (k,v) <- Map.toList kvs ]+ Dict kvs -> pure . Vec . Seq.fromList $ [Vec (Seq.fromList [k, v]) | (k,v) <- Map.toList kvs ]_ -> throwErrorHere $ TypeError "seq: can only be used on a list, vector or dictionary"), ( "to-json"@@ -403,6 +440,13 @@ purePrimFns = fromList $ allDocs $[_, _, _] -> throwErrorHere $ OtherError "verify-signature: message must be a string"xs -> throwErrorHere $ WrongNumberOfArgs "verify-signature" 3 (length xs))+ , ( "public-key?"+ , [md|Checks if a value represents a valid public key.|]+ , oneArg "public-key?" $+ \v -> case fromRad v of+ Right (_ :: PublicKey) -> pure $ Boolean True+ Left _ -> pure $ Boolean False+ ), ( "uuid?", [md|Checks if a string has the format of a UUID.|], oneArg "uuid?" $ \casediff --git a/test/spec/Radicle/Tests.hs b/test/spec/Radicle/Tests.hsindex 23cd9ae..a5cb56a 100644--- a/test/spec/Radicle/Tests.hs+++ b/test/spec/Radicle/Tests.hs@@ -71,6 +71,10 @@ test_eval =let prog = [s|(tail (list #t #f #t))|]prog `succeedsWith` List [Boolean False, Boolean True]+ , testCase "'drop' drops" $ do+ "(drop 1 (list #t #f #t))" `succeedsWith` List [Boolean False, Boolean True]+ "(drop 2 [1 2 3 4])" `succeedsWith` toRad [3 :: Int, 4]+, testCase "'nth' extracts elements from lists and vectors" $ do"(nth 0 [0 1 2])" `succeedsWith` Number 0"(nth 1 [0 1 2])" `succeedsWith` Number 1@@ -96,6 +100,11 @@ test_eval =-- Either evaluation failed or their equal.counterexample prog $ isLeft res || res == Right (Boolean False)+ , testCase "'integral?' returns true for whole numbers and false fractional numbers" $ do+ "(integral? 0)" `succeedsWith` Boolean True+ "(integral? 1)" `succeedsWith` Boolean True+ "(integral? 3.14159265359)" `succeedsWith` Boolean False+, testCase "'member?' returns true if list contains element" $ dolet prog = [s|(member? #t (list #f #t))|]prog `succeedsWith` Boolean True@@ -137,6 +146,14 @@ test_eval ="Got:\n" <> prettyEither rescounterexample (toS info) $ res == expected+ , testProperty "'string-length' returns length of a string" $ \x -> do+ let prog = "(string-length " <> renderPrettyDef (asValue (String x)) <> ")"+ res = runTest' prog+ expected = Right $ Number $ fromIntegral $ T.length x+ info = "Expected:\n" <> prettyEither (Right (String x)) <>+ "Got:\n" <> prettyEither (runTest' prog)+ counterexample (toS info) $ res == expected+, testCase "'foldl' foldls the list" $ dolet prog = [s|(foldl (fn [x y] (- x y)) 0 (list 1 2 3))|]prog `succeedsWith` Number (-6)