A peer-to-peer stack for code collaboration
Implement Radicle Daemon
James Henri Haydonf42ed2ef1e26
diff --git a/.stylish-haskell.yaml b/.stylish-haskell.yamlindex f157e71..f791b62 100644--- a/.stylish-haskell.yaml+++ b/.stylish-haskell.yaml@@ -203,4 +203,6 @@ language_extensions:- TupleSections- TypeFamilies- TypeOperators+ - NamedFieldPuns- TypeApplications+ - ViewPatternsdiff --git a/.weeder.yaml b/.weeder.yamlindex f883d5c..f07fedd 100644--- a/.weeder.yaml+++ b/.weeder.yaml@@ -2,11 +2,6 @@- name: radicle- section:- name: test:spec- - message:- - name: Redundant build-depends entry- - depends:- # Required as compiler plugin- - tasty-discover- message:# BEGIN export all for ease of debugging- name: Weeds exporteddiff --git a/bin/rad-ipfs-daemon b/bin/rad-ipfs-daemonindex 01ccea8..34ec828 100755--- a/bin/rad-ipfs-daemon+++ b/bin/rad-ipfs-daemon@@ -24,4 +24,4 @@ ipfs config Addresses.API "/ip4/127.0.0.1/tcp/9301"ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/9302"ipfs config --json Addresses.Swarm '["/ip4/0.0.0.0/tcp/9303", "/ip6/::/tcp/9303"]'-ipfs daemon "$@"+ipfs daemon --enable-pubsub-experiment "$@"diff --git a/exe/Daemon.hs b/exe/Daemon.hsnew file mode 100644index 0000000..4279149--- /dev/null+++ b/exe/Daemon.hs@@ -0,0 +1,583 @@+-- | The radicle-daemon; a long-running background process which+-- materialises the state of remote IPFS machines on the user's PC, and+-- writes to those IPFS machines the user is an owner of.+--+-- See+-- <https://github.com/oscoin/radicle/blob/master/rfcs/0003-radicle-daemon.rst+-- the RFC>.+module Daemon where++import Protolude hiding (fromStrict, option, poll)++import qualified Data.Aeson as A+import Data.ByteString.Lazy (fromStrict)+import qualified Data.Map.Strict as Map+import qualified Data.Sequence as Seq+import qualified Data.Text as T+import qualified Data.Time.Clock.System as Time+import Network.Wai.Handler.Warp (run)+import Options.Applicative+import Servant+import System.Directory (doesFileExist)++import Radicle.Daemon.Common hiding (logInfo)+import qualified Radicle.Daemon.Common as Common+import qualified Radicle.Internal.UUID as UUID++import Radicle hiding (Env)+import Radicle.Daemon.HttpApi+import Radicle.Daemon.Ipfs+import qualified Radicle.Internal.CLI as Local+import qualified Radicle.Internal.ConcurrentMap as CMap+++-- TODO(james): Check that the IPFS functions are doing all the+-- necessary pinning.++-- * Types++data MachineError+ = InvalidInput (LangError Value)+ | IpfsError IpfsError+ | AckTimeout+ | DaemonError Text+ | MachineAlreadyCached+ | MachineNotCached++data Error+ = MachineError MachineId MachineError+ | CouldNotCreateMachine Text++displayError :: Error -> (Text, [(Text,Text)])+displayError = \case+ CouldNotCreateMachine e -> ("Could not create IPFS machine", [("error", e)])+ MachineError id e -> let mid = ("machine-id", getMachineId id) in+ case e of+ InvalidInput err -> ("Invalid radicle input", [mid, ("error", renderCompactPretty err)])+ DaemonError err -> ("Internal error", [mid, ("error", err)])+ IpfsError e' -> case e' of+ IpfsDaemonError err -> ("There was an error using IPFS", [mid, ("error", toS (displayException err))])+ InternalError err -> ("Internal error", [mid, ("error", err)])+ NetworkError err -> ("There was an error communicating with the IPFS daemon", [mid, ("error", err)])+ AckTimeout -> ("The writer appears to be offline", [mid])+ MachineAlreadyCached -> ("Tried to add already cached machine", [mid])+ MachineNotCached -> ("Machine was not found in cache", [mid])++logDaemonError :: MonadIO m => Error -> m ()+logDaemonError (displayError -> (m, xs)) = logErr m xs++type Follows = Map MachineId ReaderOrWriter++-- * Main++data Opts = Opts+ { port :: Int+ -- TODO(james): temporary, just for testing.+ , filePrefix :: Text+ , debug :: Bool+ }++opts :: Parser Opts+opts = Opts+ <$> option auto+ ( long "port"+ <> help "daemon port"+ <> metavar "PORT"+ <> showDefault+ <> value 8909+ )+ <*> strOption+ ( long "filePrefix"+ <> help "file prefix"+ <> metavar "PREFIX"+ <> showDefault+ <> value ""+ )+ <*> switch+ ( long "debug"+ <> help "enable debug logging"+ <> showDefault+ )++type FollowFileLock = MVar ()++data LogLevel = Normal | Debug+ deriving (Eq, Ord)++data Env = Env+ { followFileLock :: FollowFileLock+ , followFile :: FilePath+ , machines :: IpfsMachines+ , logLevel :: LogLevel+ }++newtype Daemon a = Daemon { fromDaemon :: ExceptT Error (ReaderT Env IO) a }+ deriving (Functor, Applicative, Monad, MonadError Error, MonadIO, MonadReader Env)++runDaemon :: Env -> Daemon a -> IO (Either Error a)+runDaemon env (Daemon x) = runReaderT (runExceptT x) env++main :: IO ()+main = do+ Opts{..} <- execParser allOpts+ followFileLock <- newMVar ()+ followFile <- Local.getRadicleFile (toS filePrefix <> "daemon-follows")+ machines <- Chains <$> CMap.empty+ let env = Env{ logLevel = if debug then Debug else Normal, ..}+ follows <- readFollowFileIO followFileLock followFile+ initRes <- runDaemon env (init follows)+ case initRes of+ Left err -> logErr "Init failed" [] >> logDaemonError err+ Right _ -> do+ polling <- async $ initPolling env+ let app = serve daemonApi (server env)+ Common.logInfo "Start listening" [("port", show port)]+ serv <- async $ run port app+ exc <- waitEitherCatchCancel polling serv+ case exc of+ Left (Left err) -> logErr "Polling failed with an exception" [("error", toS (displayException err))]+ Left (Right void') -> absurd void'+ Right (Left err) -> logErr "Server failed with an exception" [("error", toS (displayException err))]+ Right (Right ()) -> logErr "Server stopped (this should not happen)" []+ where+ allOpts = info (opts <**> helper)+ ( fullDesc+ <> progDesc "Run the radicle daemon"+ <> header "radicle-daemon"+ )++type IpfsMachine = Chain MachineId MachineEntryIndex TopicSubscription+type IpfsMachines = Chains MachineId MachineEntryIndex TopicSubscription++server :: Env -> Server DaemonApi+server env = hoistServer daemonApi nt daemonServer+ where+ daemonServer :: ServerT DaemonApi Daemon+ daemonServer = (machineEndpoints :<|> newMachine) :<|> pure swagger+ machineEndpoints id = query id :<|> send id++ nt :: Daemon a -> Handler a+ nt d = do+ x_ <- liftIO $ runDaemon env d+ case x_ of+ Left err -> throwError (toServantErr err)+ Right x -> pure x++ toServantErr err@(MachineError _ e) = case e of+ InvalidInput e' -> err400 { errBody = fromStrict $ encodeUtf8 (renderPrettyDef e') }+ AckTimeout -> err504 { errBody = "The writer for this IPFS machine does not appear to be online." }+ _ -> case displayError err of+ (msg, _) -> err500 { errBody = toS msg }+ toServantErr (CouldNotCreateMachine err) = err500 { errBody = "Could not create IPFS machine: " <> toS err }++-- * Init++-- | Initiate machines according to follow file.+init :: Follows -> Daemon ()+init follows = traverse_ initMachine (Map.toList follows)+ where+ initMachine (id, Reader) = initAsReader id+ initMachine (id, Writer) = initAsWriter id++-- * Endpoints++-- | Create a new IPFS machine and initialise the daemon as as the+-- /writer/.+newMachine :: Daemon NewResponse+newMachine = do+ id <- create+ sub <- ipfs id $ initSubscription id+ logInfo Normal "Created new IPFS machine" [("machine-id", getMachineId id)]+ m <- liftIO $ emptyMachine id Writer sub+ insertNewMachine m+ actAsWriter m+ writeFollowFile+ pure (NewResponse id)+ where+ create = Daemon $ mapExceptT (lift . fmap (first CouldNotCreateMachine)) $ createMachine++-- | Evaluate an expression against a cached machine. The resulting+-- state is always discarded, and the expression is never sent to the+-- writer. Only used for local queries.+--+-- Hitting this endpoint will turn on high-frequency polling for a+-- fixed amount of time.+query :: MachineId -> QueryRequest -> Daemon QueryResponse+query id (QueryRequest v) = do+ m <- checkMachineLoaded id+ bumpPolling id+ case fst <$> runIdentity $ runLang (chainState m) $ eval v of+ Left err -> throwError $ MachineError id (InvalidInput err)+ Right rv -> do+ logInfo Normal "Query success:" [("result", renderCompactPretty rv)]+ pure (QueryResponse rv)++-- | Write a new expression to an IPFS machine.+--+-- - If the daemon is the writer for the machine, it will write the+-- new inputs to IPFS and then send out a notification on the+-- machine's pubsub topic.+--+-- - If the daemon is a reader for the machine, it will request the+-- machine's writer daemon to perform the write, and wait for an+-- ack.+send :: MachineId -> SendRequest -> Daemon SendResponse+send id (SendRequest expressions) = do+ mode_ <- machineMode+ case mode_ of+ Just (Writer, _) -> do+ results <- writeInputs id expressions Nothing+ pure SendResponse{..}+ Just (Reader, sub) -> requestInput sub+ Nothing -> do+ m <- initAsReader id+ requestInput (chainSubscription m)+ where+ requestInput sub = do+ nonce <- liftIO $ UUID.uuid+ let isResponse = \case+ New InputsApplied{ nonce = Just nonce' } | nonce' == nonce -> True+ _ -> False+ asyncMsg <- liftIO $ async $ subscribeOne sub ackWaitTime isResponse (logNonDecodableMsg id)+ ipfs id $ publish id (Submit SubmitInputs{..})+ logInfo Debug+ "Sent input request to writer"+ [ ("machine-id", getMachineId id)+ , ("expressions", prValues expressions) ]+ msg_ <- liftIO $ waitCatch asyncMsg+ case msg_ of+ Right (Just (New InputsApplied{results})) -> do+ logInfo Normal+ "Writer accepted input request"+ [ ("machine-id", getMachineId id)+ , ("results", prValues results)+ ]+ bumpPolling id+ pure SendResponse{..}+ Right Nothing -> throwError $ MachineError id AckTimeout+ Right _ -> throwError $ MachineError id (DaemonError "Didn't filter machine topic messages correctly.")+ Left err -> throwError $ MachineError id (DaemonError (toS (displayException err)))++ machineMode = fmap (liftA2 (,) chainMode chainSubscription) <$> lookupMachine id+ prValues = T.intercalate "," . (renderCompactPretty <$>)++-- * Helpers++logInfo :: LogLevel -> Text -> [(Text,Text)] -> Daemon ()+logInfo l msg infos = do+ l' <- asks logLevel+ if l <= l'+ then Common.logInfo msg infos+ else pure ()++withFollowFileLock :: FollowFileLock -> IO a -> IO a+withFollowFileLock lock = withMVar lock . const++readFollowFileIO :: FollowFileLock -> FilePath -> IO Follows+readFollowFileIO lock ff = withFollowFileLock lock $ do+ exists <- doesFileExist ff+ t <- if exists+ then readFile ff+ else let noFollows = toS (A.encode (Map.empty :: Follows))+ in writeFile ff noFollows $> noFollows+ case A.decode (toS t) of+ Nothing -> panic $ "Invalid daemon-follow file: could not decode " <> toS ff+ Just fs -> pure fs++writeFollowFile :: Daemon ()+writeFollowFile = do+ lock <- asks followFileLock+ Chains cMap <- asks machines+ ff <- asks followFile+ liftIO $ withFollowFileLock lock $ do+ ms <- CMap.nonAtomicRead cMap+ let fs = Map.fromList $ second chainMode <$> Map.toList ms+ writeFile ff (toS $ A.encode fs)++lookupMachine :: MachineId -> Daemon (Maybe IpfsMachine)+lookupMachine id = do+ msVar <- asks machines+ liftIO $ CMap.lookup id (getChains msVar)++-- | Given an 'MachineId', makes sure the machine is in the cache and+-- updated.+checkMachineLoaded :: MachineId -> Daemon IpfsMachine+checkMachineLoaded id = do+ m_ <- lookupMachine id+ case m_ of+ Nothing -> do+ -- In this case we have not seen the machine before so we act as+ -- a reader.+ m <- initAsReader id+ writeFollowFile+ pure m+ Just m -> case chainMode m of+ Writer -> pure m+ Reader -> refreshAsReader id+ -- TODO(james): For the moment we will just force a+ -- refresh. Later consider brining back the check to see if+ -- the machine was very recently updated.++ -- do+ -- delta <- liftIO $ sinceLastUpdate m+ -- -- If machine is half a second fresh, then return it.+ -- if delta < 500+ -- then pure m+ -- else refreshAsReader id++logNonDecodableMsg :: MachineId -> Text -> IO ()+logNonDecodableMsg (MachineId id) bad =+ Common.logInfo "Non-decodable message on machine's pubsub topic" [("machine-id", id), ("message", bad)]++daemonHandler :: Env -> MachineId -> (Message -> Daemon ()) -> Either Text Message -> IO ()+daemonHandler _ id _ (Left bad) = logNonDecodableMsg id bad+daemonHandler env _ h (Right msg) = do+ x_ <- runDaemon env (h msg)+ case x_ of+ Left err -> logDaemonError err+ Right _ -> pure ()++-- | Loads a machine fresh from IPFS.+loadMachine :: ReaderOrWriter -> MachineId -> Daemon IpfsMachine+loadMachine mode id = do+ (idx, is) <- ipfs id $ machineInputsFrom id Nothing+ sub <- ipfs id $ initSubscription id+ m <- liftIO $ emptyMachine id mode sub+ (m', _) <- addInputs is (pure idx) (const (pure ())) m+ insertNewMachine m'+ pure m'++-- | Add inputs to a cached machine.+addInputs+ :: [Value]+ -- ^ Inputs to add.+ -> Daemon MachineEntryIndex+ -- ^ Determine the new index.+ -> ([Value] -> Daemon ())+ -- ^ Performed after the chain is cached.+ -> IpfsMachine+ -> Daemon (IpfsMachine, ([Value], MachineEntryIndex))+ -- ^ Returns the updated machine, the results and the new index.+addInputs is getIdx after m =+ case advanceChain m is of+ Left err -> throwError $ MachineError (chainName m) (InvalidInput err)+ Right (rs, newState) -> do+ idx <- getIdx+ t <- liftIO $ Time.getSystemTime+ let news = Seq.fromList $ zip is rs+ m' = m { chainState = newState+ , chainLastIndex = Just idx+ , chainEvalPairs = chainEvalPairs m Seq.>< news+ , chainLastUpdated = t+ }+ after rs+ pure (m', (rs, idx))++ipfs :: MachineId -> ExceptT IpfsError IO a -> Daemon a+ipfs id = Daemon . mapExceptT (lift . fmap (first (MachineError id . IpfsError)))++-- | Do some high-freq polling for a while.+bumpPolling :: MachineId -> Daemon ()+bumpPolling id = do+ modifyMachine id $ \m' -> pure (m' { chainPolling = highFreq }, () )+ logInfo Debug "Reset to high-frequency polling" [("machine-id", getMachineId id)]++-- | Insert a new machine into the cache. Errors if the machine is+-- already cached.+insertNewMachine :: IpfsMachine -> Daemon ()+insertNewMachine m = do+ msVar <- asks machines+ inserted_ <- liftIO $ CMap.insertNew id m (getChains msVar)+ case inserted_ of+ Just () -> pure ()+ Nothing -> throwError (MachineError id MachineAlreadyCached)+ where+ id = chainName m++-- | Modify a machine that is already in the cache. Errors if the+-- machine isn't in the cache already.+modifyMachine :: MachineId -> (IpfsMachine -> Daemon (IpfsMachine, a)) -> Daemon a+modifyMachine id f = do+ env <- ask+ res <- liftIO $ CMap.modifyExistingValue id (getChains (machines env)) $ \m -> do+ x <- runDaemon env (f m)+ pure $ case x of+ Left err -> (m, Left err)+ Right (m', y) -> (m', Right y)+ case res of+ Nothing -> throwError (MachineError id MachineNotCached)+ Just (Left err) -> throwError err+ Just (Right y) -> pure y++emptyMachine :: MachineId -> ReaderOrWriter -> TopicSubscription -> IO IpfsMachine+emptyMachine id mode sub = do+ t <- Time.getSystemTime+ pure Chain{ chainName = id+ , chainState = pureEnv+ , chainEvalPairs = mempty+ , chainLastIndex = Nothing+ , chainMode = mode+ , chainSubscription = sub+ , chainLastUpdated = t+ , chainPolling = highFreq+ }++-- ** Reader++-- | Load a machine in /reader mode/ and return it.+--+-- Loads the machines input log from IPFS and listens for machine+-- updates on pubsub.+--+-- The machine is added to the deamons machine cache.+initAsReader :: MachineId -> Daemon IpfsMachine+initAsReader id = do+ m <- loadMachine Reader id+ env <- ask+ _ <- liftIO $ addHandler (chainSubscription m) (daemonHandler env id onMsg)+ logInfo Normal "Following as reader" [ ("machine-id", getMachineId id)+ , ("current-input-index", show (chainLastIndex m)) ]+ pure m+ where+ onMsg :: Message -> Daemon ()+ onMsg = \case+ New InputsApplied{..} -> do+ _ <- refreshAsReader id+ -- TODO(james): decide if this should bump polling. If this is+ -- a very active chain this will mean the polling is always+ -- high-frequency.+ bumpPolling id+ _ -> pure ()++-- | Updates a machine in the cache with the new inputs pulled from IPFS.+refreshAsReader :: MachineId -> Daemon IpfsMachine+refreshAsReader id = modifyMachine id $ \m -> do+ let currentIdx = chainLastIndex m+ (newIdx, is) <- ipfs id $ machineInputsFrom (chainName m) currentIdx+ if currentIdx == Just newIdx+ then do+ logInfo Debug+ "Reader is already up to date"+ [mid, ("input-index", show newIdx)]+ pure (m, m)+ else do+ (m', _) <- addInputs is (pure newIdx) (const (pure ())) m+ logInfo Debug+ "Updated reader"+ [mid, ("n", show (length is)), ("input-index", show newIdx)]+ pure (m', m')+ where+ mid = ("machine-id", getMachineId id)++-- ** Writer++initAsWriter :: MachineId -> Daemon IpfsMachine+initAsWriter id = do+ m <- loadMachine Writer id+ actAsWriter m+ pure m++-- | Subscribes the daemon to the machine's pubsub topic to listen for+-- input requests.+actAsWriter :: IpfsMachine -> Daemon ()+actAsWriter m = do+ env <- ask+ _ <- liftIO $ addHandler (chainSubscription m) (daemonHandler env id onMsg)+ logInfo Normal "Acting as writer" [("machine-id", getMachineId id)]+ where+ id = chainName m+ onMsg = \case+ Submit SubmitInputs{..} -> writeInputs id expressions (Just nonce) >> pure ()+ _ -> pure ()++-- | Write and evaluate inputs in a machine we control.+--+-- Returns the outputs generated by evaluating the inputs.+--+-- Sends out a 'InputsApplied' message over pubsub that includes the+-- outputs and the given nonce.+writeInputs :: MachineId -> [Value] -> Maybe Text -> Daemon [Value]+writeInputs id is nonce = do+ (rs, idx) <- modifyMachine id $ addInputs is write pub+ logInfo Debug "Wrote inputs to IPFS" [ ("machine-id", getMachineId id)+ , ("new-input-index", show idx) ]+ pure rs+ where+ write = ipfs id $ writeIpfs id is+ pub results = ipfs id $ publish id (New InputsApplied{..})++-- * Polling++-- | Fetch and apply new inputs for all machines in reader mode.+poll :: Daemon ()+poll = do+ msVar <- asks machines+ ms <- liftIO $ CMap.nonAtomicRead (getChains msVar)+ traverse_ pollMachine ms+ where+ pollMachine :: IpfsMachine -> Daemon ()+ pollMachine m@Chain{chainMode = Reader, ..} = do+ delta <- liftIO $ sinceLastUpdate m+ let (shouldPoll, newPoll) =+ case chainPolling of+ HighFreq more ->+ let more' = more - delta+ in if more' > 0+ then (True, HighFreq more')+ else (False, LowFreq)+ -- Low frequency polling is every 10 seconds.+ LowFreq -> (delta > lowFreqPollPeriod, LowFreq)+ when shouldPoll $ do+ logInfo Debug "Polling.." [("machine-id", getMachineId chainName)]+ refreshAsReader chainName >> pure ()+ modifyMachine chainName $ \m' -> pure (m' { chainPolling = newPoll }, () )+ pollMachine _ = pure ()++-- | Returns the amount of time since the last time the machine was+-- updated.+sinceLastUpdate :: IpfsMachine -> IO Milliseconds+sinceLastUpdate m = timeDelta (chainLastUpdated m) <$> Time.getSystemTime+ where+ timeDelta x y =+ case (trunc x, trunc y) of+ (Time.MkSystemTime s n, Time.MkSystemTime s' n') -> 1000 * (s' - s) + fromIntegral (n' - n) `div` 1000000+ trunc = Time.truncateSystemTimeLeapSecond++-- | Polling loop that looks for changes on all loaded reader machines+-- and updates them.+initPolling :: Env -> IO Void+initPolling env = do+ threadDelay (millisToMicros highFreqPollPeriod)+ res <- runDaemon env poll+ -- If polling encounters an error it should log it but continue.+ -- Later we might detect some errors as critical and halt the+ -- daemon.+ case res of+ Left err -> logDaemonError err+ Right _ -> pure ()+ initPolling env++-- * Timings++type Milliseconds = Int64++millisToMicros :: Milliseconds -> Int+millisToMicros n = 1000 * fromIntegral n++-- | High-frequency polling happens once every half-second.+highFreqPollPeriod :: Milliseconds+highFreqPollPeriod = 500++-- | Low-frequency polling happens once every 10 seconds.+lowFreqPollPeriod :: Milliseconds+lowFreqPollPeriod = 10 * 1000++-- | The amount of time a reader will wait for a response message from+-- a writer: 8 seconds.+ackWaitTime :: Milliseconds+ackWaitTime = 8 * 1000++-- | The amount of time a machine does high-frequency polling for+-- before it returns to low-frequency polling: 10 minutes.+highFreq :: Polling+highFreq = HighFreq (10 * 60 * 1000)diff --git a/exe/ReferenceDoc.hs b/exe/ReferenceDoc.hsindex 5b7d7cc..73dd0cc 100644--- a/exe/ReferenceDoc.hs+++ b/exe/ReferenceDoc.hs@@ -13,6 +13,7 @@ import qualified Data.Map.Strict as Mapimport qualified Data.Text as Timport Data.Yaml hiding (Value)import Radicle+import qualified Radicle.Daemon.HttpApi as Daemonimport Radicle.Internal.Coreimport qualified Radicle.Internal.Doc as Docimport Radicle.Internal.Identifier@@ -43,6 +44,7 @@ main = dolet env = bindingsEnv $ bindingsFromRadicle res `lPanic` "Couldn't convert radicle state."let rst = runPure (writeRST Default.def $ Pandoc nullMeta (doc content env)) `lPanic` "Couldn't generate RST"writeFile "docs/source/reference.rst" rst+ writeFile "docs/source/daemon-api.yaml" (toS (encode Daemon.swagger))wheredoc :: Content -> Env Value -> [Block]diff --git a/exe/Server.hs b/exe/Server.hsindex 26f373a..5e8aa16 100644--- a/exe/Server.hs+++ b/exe/Server.hs@@ -8,7 +8,7 @@ import qualified Data.Aeson as Aimport Data.ByteString.Lazy (fromStrict)import qualified Data.Map as Mapimport qualified Data.Sequence as Seq-import qualified Data.Text as T+import qualified Data.Time.Clock.System as Timeimport Database.PostgreSQL.Simple(ConnectInfo(..), Connection, connect)import Network.Wai.Handler.Warp (run)@@ -48,6 +48,8 @@ import Servant)import Radicle+import Radicle.Daemon.Common+import qualified Radicle.Internal.ConcurrentMap as CMapimport Radicle.Internal.MachineBackend.EvalServerimport Server.DB@@ -56,16 +58,6 @@ import Server.DBnewtype Exprs = Exprs { getExprs :: Seq Value }deriving (Generic, Eq, Ord)-data Chain = Chain- { chainName :: Text- , chainState :: Bindings (PrimFns Identity)- , chainEvalPairs :: Seq.Seq (Value, Value)- } deriving (Generic)---- For efficiency we might want keys to also be mvars, but efficiency doesn't--- matter for a test server.-newtype Chains = Chains { getChains :: MVar (Map Text Chain) }-type ServerApi= "chains" :> Capture "chain" Text :> (ChainSubmitEndpoint :<|> ChainSinceEndpoint):<|> "outputs" :> Capture "chain" Text :> Get '[JSON] [Maybe A.Value]@@ -74,38 +66,39 @@ type ServerApiserverApi :: Proxy ServerApiserverApi = Proxy+type HttpChains = Chains Text Int ()-- * Helpers-- | Returns the ID of the last inserted expression which is the chain-- length minus one.-insertExpr :: Connection -> Chains -> Text -> [Value] -> IO (Either Text Int)-insertExpr conn chains name vals = modifyMVar (getChains chains) $ \c -> do- let maybeChain = Map.lookup name c+insertExpr :: Connection -> HttpChains -> Text -> [Value] -> IO (Either Text Int)+insertExpr conn chains name vals = CMap.modifyValue name (getChains chains) $ \maybeChain -> dochain <- case maybeChain ofNothing -> dologInfo "Creating new machine" [("machine", name)]- pure $ Chain name pureEnv mempty+ t <- Time.getSystemTime+ pure $ Chain name pureEnv mempty Nothing Writer () t LowFreqJust chain' -> pure chain'- let (r :: Either (LangError Value) [Value], newSt) = runIdentity- $ runLang (chainState chain)- $ traverse eval vals- case r of+ case advanceChain chain vals ofLeft e -> dologInfo "Submitted expressions failed to evaluate" [("machine", name)]- pure . (c ,) . Left $ renderPrettyDef e- Right valsRes -> do+ pure (Nothing, Left $ renderPrettyDef e)+ Right (valsRes, newSt) -> dotraverse_ (insertExprDB conn name) valslet news = Seq.fromList $ zip vals valsRes+ let pairs = chainEvalPairs chain Seq.>< news+ let lastIndex = Seq.length pairs - 1let chain' = chain { chainState = newSt- , chainEvalPairs = chainEvalPairs chain Seq.>< news+ , chainEvalPairs = pairs+ , chainLastIndex = Just lastIndex}- pure (Map.insert name chain' c, Right $ Seq.length (chainEvalPairs chain') - 1)+ pure (Just chain', Right lastIndex)-- | Load the state from the DB, returning an MVar with resulting state-- and values.-loadState :: Connection -> IO Chains+loadState :: Connection -> IO HttpChainsloadState conn = docreateIfNotExists connres <- getAllDB conn@@ -117,12 +110,19 @@ loadState conn = dologInfo "Failed to load machine" [("machine", name)]panic $ show err(Right pairs, st') -> do+ t <- Time.getSystemTimelet c = Chain { chainName = name, chainState = st', chainEvalPairs = pairs+ , chainLastIndex = Just $ Seq.length pairs - 1+ , chainMode = Writer+ , chainSubscription = ()+ , chainLastUpdated = t+ , chainPolling = LowFreq}pure (name, c)- chains' <- newMVar $ Map.fromList chainPairs+ --chains' <- newMVar $ Map.fromList chainPairs+ chains' <- CMap.fromMap $ Map.fromList chainPairslogInfo "Loaded machines into memory" [("machine-count", show $ length chainPairs)]pure $ Chains chains'@@ -134,7 +134,7 @@ loadState conn = do-- second the expressions being submitted. The server either accepts all or-- rejects all expressions (though other systems may not provide this-- guarantee).-submit :: Connection -> Chains -> Text -> Values -> Handler Value+submit :: Connection -> HttpChains -> Text -> Values -> Handler Valuesubmit conn chains name (Values vals) = dologInfo "Submitted expressions" [("machine", name), ("expression-count", show $ length vals)]res <- liftIO $ insertExpr conn chains name vals@@ -149,31 +149,16 @@ since conn name index = dologInfo "Requested expressions" [("machine", name), ("from-index", show index)]liftIO $ Values <$> getSinceDB conn name index-jsonOutputs :: Chains -> Text -> Handler [Maybe A.Value]+jsonOutputs :: HttpChains -> Text -> Handler [Maybe A.Value]jsonOutputs st name = do- chains <- liftIO . readMVar $ getChains st- pure $ case Map.lookup name chains of+ maybeChain <- liftIO $ CMap.lookup name (getChains st)+ pure $ case maybeChain ofNothing -> []Just chain -> toList $ maybeJson . snd <$> chainEvalPairs chainstatic :: Server Rawstatic = serveDirectoryFileServer "static/"--- | Log a message to @stdout@.------ The second argument is a list of key-value pairs that will be joined with "="--- and appended to the message.------ @--- logInfo "the message" [("foo", "5)]--- -- prints "INFO the message foo=5"--- @-logInfo :: MonadIO m => Text -> [(Text, Text)] -> m ()-logInfo msg dat = do- putStrLn $ "INFO " <> msg <> datString- where- datString = T.intercalate "" $ map (\(key, val) -> " " <> key <> "=" <> val) dat--- * Optsdata Opts = Opts@@ -247,7 +232,7 @@ main = do<> header "radicle-server")-server :: Connection -> Chains -> Server ServerApi+server :: Connection -> HttpChains -> Server ServerApiserver conn chains = chainEndpoints :<|> jsonOutputs chains :<|> staticwherechainEndpoints chainName = submit conn chains chainName :<|> since conn chainNamediff --git a/package.yaml b/package.yamlindex 206fcc7..c1f5e28 100644--- a/package.yaml+++ b/package.yaml@@ -29,6 +29,8 @@ library:- aeson- ansi-terminal- bytestring+ - conduit+ - conduit-extra- containers- cryptonite- directory@@ -36,19 +38,24 @@ library:- generics-eot- haskeline- http-client+ - http-conduit- ipld-cid- megaparsec- microlens- mtl+ - multibase- pointed- prettyprinter- prettyprinter-ansi-terminal- process+ - resourcet- safe-exceptions- scientific- serialise- servant- servant-client+ - servant-swagger+ - swagger2- template-haskell- text- time@@ -76,7 +83,6 @@ tests:- serialise- string-qq- tasty- - tasty-discover- tasty-hunit- tasty-quickcheck- text@@ -138,8 +144,27 @@ executables:- warp- wai-extra- wai-cors+ - timeghc-options: -main-is Server+ radicle-daemon:+ main: Daemon.hs+ source-dirs: exe+ other-modules: []+ dependencies:+ - radicle+ - aeson+ - mtl+ - containers+ - bytestring+ - directory+ - optparse-applicative+ - servant-server+ - text+ - warp+ - time+ ghc-options: -main-is Daemon+radicle-doc:main: Doc.hssource-dirs: exe/@@ -205,6 +230,7 @@ default-extensions:- DeriveFunctor- DeriveGeneric- DeriveTraversable+- DuplicateRecordFields- FlexibleContexts- FlexibleInstances- FunctionalDependencies@@ -212,6 +238,7 @@ default-extensions:- LambdaCase- MultiParamTypeClasses- MultiWayIf+- NamedFieldPuns- NoImplicitPrelude- OverloadedStrings- RankNTypes@@ -221,3 +248,4 @@ default-extensions:- TupleSections- TypeFamilies- TypeOperators+- ViewPatternsdiff --git a/src/Radicle/Daemon/Common.hs b/src/Radicle/Daemon/Common.hsnew file mode 100644index 0000000..7f6c8ff--- /dev/null+++ b/src/Radicle/Daemon/Common.hs@@ -0,0 +1,66 @@+module Radicle.Daemon.Common+ ( ReaderOrWriter(..)+ , Polling(..)+ , Chain(..)+ , Chains(..)+ , logInfo+ , logErr+ , advanceChain+ ) where++import Protolude hiding (log)++import qualified Data.Aeson as A+import qualified Data.Sequence as Seq+import qualified Data.Text as T+import qualified Data.Time.Clock.System as Time++import Radicle+import qualified Radicle.Internal.ConcurrentMap as CMap++data ReaderOrWriter = Reader | Writer+ deriving (Generic)++data Polling = HighFreq Int64 | LowFreq++instance A.ToJSON ReaderOrWriter+instance A.FromJSON ReaderOrWriter++data Chain id idx subs = Chain+ { chainName :: id+ , chainState :: Bindings (PrimFns Identity)+ , chainEvalPairs :: Seq.Seq (Value, Value)+ , chainLastIndex :: Maybe idx+ , chainMode :: ReaderOrWriter+ , chainSubscription :: subs+ , chainLastUpdated :: Time.SystemTime+ , chainPolling :: Polling+ } deriving (Generic)++newtype Chains id idx subs = Chains { getChains :: CMap.CMap id (Chain id idx subs) }++-- | Log a message to @stdout@.+--+-- The second argument is a list of key-value pairs that will be joined with "="+-- and appended to the message.+--+-- @+-- log "INFO" "the message" [("foo", "5)]+-- -- prints "INFO the message foo=5"+-- @+log :: MonadIO m => Text -> Text -> [(Text, Text)] -> m ()+log typ msg dat = do+ putStrLn $ typ <> " " <> msg <> datString+ where+ datString = T.intercalate "" $ map (\(key, val) -> " " <> key <> "=" <> val) dat++logInfo, logErr :: MonadIO m => Text -> [(Text, Text)] -> m ()+logInfo = log "INFO "+logErr = log "ERROR"++advanceChain :: Chain id idx subs -> [Value] -> Either (LangError Value) ([Value], Bindings (PrimFns Identity))+advanceChain chain vals =+ let (r_, newSt) = runIdentity $ runLang (chainState chain) $ traverse eval vals+ in case r_ of+ Left e -> Left e+ Right r -> pure (r, newSt)diff --git a/src/Radicle/Daemon/HttpApi.hs b/src/Radicle/Daemon/HttpApi.hsnew file mode 100644index 0000000..b5c1627--- /dev/null+++ b/src/Radicle/Daemon/HttpApi.hs@@ -0,0 +1,123 @@+{-# OPTIONS_GHC -fno-warn-orphans #-}+module Radicle.Daemon.HttpApi+ ( QueryRequest(..)+ , QueryResponse(..)+ , SendRequest(..)+ , SendResponse(..)+ , NewResponse(..)+ , DaemonApi+ , daemonApi+ , swagger+ ) where++import qualified Prelude+import Protolude++import Data.Aeson ((.:))+import qualified Data.Aeson as A+import qualified Data.Aeson.Types as A+import Data.Swagger+import Data.Version (showVersion)+import Radicle.Daemon.Ipfs+import Servant.API+import Servant.Swagger++import qualified Paths_radicle as Radicle+import Radicle++instance FromHttpApiData MachineId where+ parseUrlPiece = Right . MachineId . toS+ parseQueryParam = parseUrlPiece++singleKey :: Prelude.String -> (a -> b) -> (A.Object -> A.Parser a) -> A.Value -> A.Parser b+singleKey tyName c k = A.withObject tyName $ k >=> pure . c++newtype QueryRequest = QueryRequest { expression :: Value }+ deriving (Generic)++instance A.ToJSON QueryRequest where+ toJSON = valueToJson . expression+instance A.FromJSON QueryRequest where+ parseJSON = singleKey "QueryRequest" QueryRequest $ (.: "expression") >=> jsonToValue++newtype QueryResponse = QueryResponse { result :: Value }+ deriving (Generic)++instance A.ToJSON QueryResponse where+ toJSON = valueToJson . result+instance A.FromJSON QueryResponse where+ parseJSON = singleKey "QueryResponse" QueryResponse $ (.: "result") >=> jsonToValue++newtype SendRequest = SendRequest { expressions :: [Value] }+ deriving (Generic)++instance A.FromJSON SendRequest where+ parseJSON = singleKey "SendRequest" SendRequest $ (.: "expressions") >=> traverse jsonToValue++newtype SendResponse = SendResponse+ { results :: [Value]+ } deriving (Generic)++instance A.ToJSON SendResponse where+ toJSON = A.toJSON . fmap valueToJson . (results :: SendResponse -> [Value])++newtype NewResponse = NewResponse+ { machineId :: MachineId+ } deriving (Generic)++instance A.ToJSON NewResponse++-- * APIs++type Query = "query" :> ReqBody '[JSON] QueryRequest :> Post '[JSON] QueryResponse+type Send = "send" :> ReqBody '[JSON] SendRequest :> Post '[JSON] SendResponse+type New = "new" :> Post '[JSON] NewResponse++type Machines = "machines" :> Capture "machineId" MachineId :> ( Query :<|> Send ) :<|> New+type Docs = "docs" :> Get '[JSON] Swagger++type DaemonApi = "v0" :> ( Machines :<|> Docs )++daemonApi :: Proxy DaemonApi+daemonApi = Proxy++-- * Docs++instance ToParamSchema MachineId where+ toParamSchema _ = mempty { _paramSchemaType = SwaggerString }+instance ToSchema MachineId where+ declareNamedSchema pxy = pure $+ NamedSchema (Just "MachineId") $+ mempty { _schemaDescription = Just "The ID (i.e. IPNS name) of an IPFS machine."+ , _schemaParamSchema = toParamSchema pxy }++instance ToSchema Value where+ declareNamedSchema _ = pure $+ NamedSchema (Just "RadicleValue") $+ mempty { _schemaDescription = Just "A radicle value formatted according to radicle's S-expression syntax."+ , _schemaParamSchema = mempty {_paramSchemaType = SwaggerString } }++instance ToSchema QueryRequest+instance ToSchema QueryResponse+instance ToSchema SendRequest+instance ToSchema SendResponse+instance ToSchema NewResponse++instance ToSchema Swagger where+ declareNamedSchema _ = pure $+ NamedSchema (Just "Swagger") $+ mempty { _schemaDescription = Just "This swagger spec."+ , _schemaParamSchema = mempty { _paramSchemaType = SwaggerObject }}++swagger :: Swagger+swagger = swag { _swaggerInfo = inf }+ where+ inf = mempty { _infoTitle = "Radicle daemon"+ , _infoDescription = Just desc+ , _infoVersion = toS $ showVersion Radicle.version+ }+ swag = toSwagger daemonApi+ desc =+ "The radicle-daemon; a long-running background process which\+ \ materialises the state of remote IPFS machines on the users\+ \ PC, and writes to those IPFS machines the user is an owner of."diff --git a/src/Radicle/Daemon/Ipfs.hs b/src/Radicle/Daemon/Ipfs.hsnew file mode 100644index 0000000..e441814--- /dev/null+++ b/src/Radicle/Daemon/Ipfs.hs@@ -0,0 +1,211 @@+module Radicle.Daemon.Ipfs+ ( IpfsError(..)+ , MachineId(..)+ , Message(..)+ , InputsApplied(..)+ , SubmitInputs(..)+ , writeIpfs+ , publish+ , machineInputsFrom+ , createMachine+ , Ipfs.MachineEntryIndex+ , TopicSubscription+ , initSubscription+ , subscribeOne+ , addHandler+ , valueToJson+ , jsonToValue+ ) where++import Protolude hiding (bracket, catches)++import Control.Exception.Safe+import Control.Monad.Fail+import Data.Aeson (decodeStrict, (.:), (.=))+import qualified Data.Aeson as Aeson+import qualified Data.Aeson.Types as Aeson+import qualified Data.HashMap.Strict as HashMap+import qualified Data.Map.Strict as Map+import qualified Data.Sequence as Seq+import qualified Data.Unique as Unique+import qualified Network.HTTP.Client as Http++import Radicle+import qualified Radicle.Internal.MachineBackend.Ipfs as Ipfs+import qualified Radicle.Internal.UUID as UUID+import qualified Radicle.Ipfs as Ipfs++jsonToValue :: Aeson.Value -> Aeson.Parser Value+jsonToValue = Aeson.withText "Value" $ \t -> do+ case parse "[daemon]" t of+ Left err -> fail $ "failed to parse Radicle expression: " <> show err+ Right v -> pure v++valueToJson :: Value -> Aeson.Value+valueToJson = Aeson.String . renderCompactPretty++-- * Types++newtype MachineId = MachineId { getMachineId :: Text }+ deriving (Show, Eq, Ord, Generic)++instance Aeson.ToJSONKey MachineId where+ toJSONKey = Aeson.toJSONKeyText getMachineId+instance Aeson.FromJSONKey MachineId where+ fromJSONKey = Aeson.FromJSONKeyText MachineId+instance Aeson.ToJSON MachineId where+ toJSON (MachineId id) = Aeson.String id+instance Aeson.FromJSON MachineId where+ parseJSON js = MachineId <$> Aeson.parseJSON js++data IpfsError+ = IpfsDaemonError Ipfs.IpfsException+ | InternalError Text+ | NetworkError Text++-- | Messages sent on a machine's IPFS pubsub topic.+data Message = New InputsApplied | Submit SubmitInputs++instance Aeson.FromJSON Message where+ parseJSON = Aeson.withObject "Message" $ \o -> do+ typ :: Text <- o .: "type"+ case typ of+ "new_inputs" -> do+ nonce <- o .: "nonce"+ results <- traverse jsonToValue =<< o .: "results"+ pure $ New InputsApplied{..}+ "request_inputs" -> do+ nonce <- o .: "nonce"+ expressions <- traverse jsonToValue =<< o .: "expressions"+ pure $ Submit SubmitInputs{..}+ _ -> fail "Messages on a machine's pubsub topic must be of type 'new_inputs' or 'request_inputs'."++instance Aeson.ToJSON Message where+ toJSON = \case+ New InputsApplied{..} -> Aeson.object+ [ "type" .= ("new_inputs" :: Text)+ , "nonce" .= nonce+ , "results" .= (valueToJson <$> results)+ ]+ Submit SubmitInputs{..} -> Aeson.object+ [ "type" .= ("request_inputs" :: Text)+ , "nonce" .= nonce+ , "expressions" .= (valueToJson <$> expressions)+ ]++-- | Message sent to signal a new input has been added to the machine.+data InputsApplied = InputsApplied+ { nonce :: Maybe Text+ , results :: [Value]+ }++-- | Message sent to request the writer add inputs to the machine.+data SubmitInputs = SubmitInputs+ { nonce :: Text+ , expressions :: [Value]+ }++-- * Ipfs helpers++-- TODO: All the error handling w.r.t interactions with IPFS should be+-- done in "Radicle.Ipfs".+safeIpfs :: forall a. IO a -> ExceptT IpfsError IO a+safeIpfs io = ExceptT $ liftIO $+ catches (Right <$> io)+ [ Handler $ pure . Left . IpfsDaemonError+ , Handler httpHdlr+ ]+ where+ httpHdlr e = pure $ Left $ case e of+ Http.HttpExceptionRequest _+ (Http.StatusCodeException _+ (decodeStrict ->+ Just (Aeson.Object (HashMap.lookup "Message" ->+ Just (Aeson.String msg))))) -> NetworkError msg+ _ -> InternalError (toS (displayException e))+++-- | Write some inputs to an IPFS machine.+-- TODO: we might want to make this safer by taking the expected head+-- MachineEntryIndex as an argument.+writeIpfs :: MachineId -> [Value] -> ExceptT IpfsError IO Ipfs.MachineEntryIndex+writeIpfs (MachineId id) vs = safeIpfs $ Ipfs.sendIpfs id (Seq.fromList vs)++newtype Topic = Topic { getTopic :: Text }++-- | The IPFS pubsub topic associated to a machine.+machineTopic :: MachineId -> Topic+machineTopic (MachineId id) = Topic ("radicle:machine:" <> id)++-- | Publish a 'Message' on a machine's IPFS pubsub topic.+publish :: MachineId -> Message -> ExceptT IpfsError IO ()+publish id msg = safeIpfs $+ Ipfs.publish (getTopic (machineTopic id)) (Aeson.encode msg)++-- | Subscribe to messages on a machine's IPFS pubsub topic.+subscribeForever :: MachineId -> MsgHandler -> IO ()+subscribeForever id messageHandler = Ipfs.subscribe topic pubsubHandler+ where+ Topic topic = machineTopic id+ pubsubHandler Ipfs.PubsubMessage{..} = messageHandler $ case decodeStrict messageData of+ Just msg -> Right msg+ Nothing -> Left (toS messageData)++-- | Get inputs of an IPFS machine from a certain index.+machineInputsFrom :: MachineId -> Maybe Ipfs.MachineEntryIndex -> ExceptT IpfsError IO (Ipfs.MachineEntryIndex, [Value])+machineInputsFrom (MachineId id) = safeIpfs . Ipfs.receiveIpfs id++-- | Create an IPFS machine and return its ID.+createMachine :: ExceptT Text IO MachineId+createMachine = ExceptT $ liftIO $ second MachineId <$> (Ipfs.ipfsMachineCreate =<< UUID.uuid)++-- * Topic subscriptions++type MsgHandler = Either Text Message -> IO ()++type RegisteredHandler = Unique.Unique++newtype TopicSubscription = TopicSubscription+ { handlers :: MVar (Map RegisteredHandler MsgHandler)+ }++-- | Given a machine ID, creates a subscription for that machine's+-- pubsub topic. This allows adding and removing handlers for messages+-- on that topic.+initSubscription :: MachineId -> ExceptT IpfsError IO TopicSubscription+initSubscription id = safeIpfs $ do+ hdlrs <- newMVar Map.empty+ _ <- async $ subscribeForever id (mainHdlr hdlrs)+ pure $ TopicSubscription hdlrs+ where+ mainHdlr hdlrs msg = do+ hs <- readMVar hdlrs+ traverse_ ($ msg) (Map.elems hs)++-- | Add a message handler to a topic subscription.+addHandler :: TopicSubscription -> MsgHandler -> IO RegisteredHandler+addHandler ts h = do+ u <- Unique.newUnique+ modifyMVar (handlers ts) (pure . (,()) . Map.insert u h)+ pure u++-- | Remove a message handler from a topic subscription.+removeHandler :: TopicSubscription -> RegisteredHandler -> IO ()+removeHandler ts u =+ modifyMVar (handlers ts) (pure . (,()) . Map.delete u)++-- | Block while waiting for a message which matches a+-- predicate. Returns @Just msg@ where @msg@ is the first message+-- passing the predicate if such a message arrives before the+-- specified amount of milliseconds. Otherwise, returns @Nothing@.+subscribeOne :: TopicSubscription -> Int64 -> (Message -> Bool) -> (Text -> IO ()) -> IO (Maybe Message)+subscribeOne sub timeout pr badMsg = do+ var <- newEmptyMVar+ let onMsg = \case+ Right msg | pr msg -> putMVar var (Just msg)+ Right _ -> pure ()+ Left bad -> badMsg bad+ bracket+ (addHandler sub onMsg)+ (removeHandler sub)+ (const $ forkIO (threadDelay (fromIntegral timeout * 1000) >> putMVar var Nothing) *> readMVar var)diff --git a/src/Radicle/Internal/CLI.hs b/src/Radicle/Internal/CLI.hsindex 81387c0..e34e16e 100644--- a/src/Radicle/Internal/CLI.hs+++ b/src/Radicle/Internal/CLI.hs@@ -1,5 +1,6 @@module Radicle.Internal.CLI- ( getHistoryFile+ ( getRadicleFile+ , getHistoryFile)where@@ -8,15 +9,18 @@ import System.Directory(XdgDirectory(..), createDirectoryIfMissing, getXdgDirectory)import System.FilePath (takeDirectory)--- | Location of the radicle history, usually--- @~/.local/share/radicle/history@.---+-- | Location of a radicle related file.-- Creates the parent directory if it does not exist.---- See 'getXdgDirectory' 'XdgData' for how @~/.local/share@ is-- determined.-getHistoryFile :: IO FilePath-getHistoryFile = do- file <- getXdgDirectory XdgData "radicle/history"+getRadicleFile :: FilePath -> IO FilePath+getRadicleFile name = do+ file <- getXdgDirectory XdgData $ "radicle/" <> namecreateDirectoryIfMissing True (takeDirectory file)pure file++-- | Location of the radicle history, usually+-- @~/.local/share/radicle/history@.+getHistoryFile :: IO FilePath+getHistoryFile = getRadicleFile "history"diff --git a/src/Radicle/Internal/ConcurrentMap.hs b/src/Radicle/Internal/ConcurrentMap.hsnew file mode 100644index 0000000..cb4a4e4--- /dev/null+++ b/src/Radicle/Internal/ConcurrentMap.hs@@ -0,0 +1,83 @@+module Radicle.Internal.ConcurrentMap+ ( empty+ , fromMap+ , lookup+ , nonAtomicRead+ , insertNew+ , modifyExistingValue+ , modifyValue+ , CMap+ ) where++import Protolude hiding (empty)++import qualified Data.Map.Strict as Map++-- | A Map which offers atomic operations on the values associated to+-- keys. Assumes that the values are independent of one another; does+-- not offer a consistent view over the values. Therefore this is best+-- used when the keys uniquely identify the resources represented by+-- the values.+newtype CMap k v = CMap (MVar (Map k (MVar v)))++-- | Create a new empty 'CMap'.+empty :: IO (CMap k v)+empty = CMap <$> newMVar Map.empty++fromMap :: Map k v -> IO (CMap k v)+fromMap m = CMap <$> (traverse newMVar m >>= newMVar)++-- | Atomically lookup a value.+lookup :: Ord k => k -> CMap k v -> IO (Maybe v)+lookup k (CMap m_) = withMVar m_ $ \m -> do+ case Map.lookup k m of+ Nothing -> pure Nothing+ Just v_ -> pure <$> readMVar v_++-- | Non-atmoically read the contents of a 'CMap'. Provides a+-- consistent shapshot of which keys were present at some+-- time. However, the values might be snapshotted at different times.+nonAtomicRead :: CMap k v -> IO (Map k v)+nonAtomicRead (CMap m_) = do+ m <- readMVar m_+ traverse readMVar m++-- | Atomically insert a key-value pair into a 'CMap' but only if the+-- key is not already a member.+insertNew :: Ord k => k -> v -> CMap k v -> IO (Maybe ())+insertNew k v (CMap m_) = modifyMVar m_ $ \m ->+ if Map.member k m+ then pure (m, Nothing)+ else do+ v_ <- newMVar v+ let m' = Map.insert k v_ m+ pure (m', Just ())++-- | Atomically modifies a value associated to a key but only if it+-- exists. There is no guarantee the value is still associated with+-- the same key, or any key, by the time the operation completes.+modifyExistingValue :: Ord k => k -> CMap k v -> (v -> IO (v, a)) -> IO (Maybe a)+modifyExistingValue k (CMap m_) f = do+ m <- readMVar m_+ case Map.lookup k m of+ Nothing -> pure Nothing+ Just v_ -> do+ x <- modifyMVar v_ f+ pure (Just x)++-- | Atomically modifies the value associated to a key.+modifyValue :: Ord k => k -> CMap k v -> (Maybe v -> IO (Maybe v, a)) -> IO a+modifyValue k (CMap m_) f = modifyMVar m_ $ \m ->+ case Map.lookup k m of+ Nothing -> do+ (maybev, x) <- f Nothing+ case maybev of+ Just v -> do+ v_ <- newMVar v+ pure (Map.insert k v_ m, x)+ Nothing -> pure (m, x)+ Just v_ -> modifyMVar v_ $ \v -> do+ (maybev', x) <- f (Just v)+ case maybev' of+ Just v' -> pure (v', (m, x))+ Nothing -> pure (v, (Map.delete k m, x))diff --git a/src/Radicle/Internal/Core.hs b/src/Radicle/Internal/Core.hsindex 2b11b87..55cc06f 100644--- a/src/Radicle/Internal/Core.hs+++ b/src/Radicle/Internal/Core.hs@@ -11,7 +11,7 @@ import Codec.Serialise (Serialise)import Control.Monad.Except(ExceptT(..), MonadError, runExceptT, throwError)import Control.Monad.State hiding (State)-import Data.Aeson (FromJSON(..), ToJSON(..))+import Data.Aeson (ToJSON(..))import qualified Data.Aeson as Aimport Data.Copointed (Copointed(..))import qualified Data.HashMap.Strict as HashMap@@ -402,18 +402,6 @@ isAtom :: Value -> Maybe IdentisAtom (Atom i) = pure iisAtom _ = Nothing-instance A.FromJSON Value where- parseJSON = \case- A.Number n -> pure $ Number (toRational n)- A.String s -> pure $ String s- A.Array ls -> List . toList <$> traverse parseJSON ls- A.Bool b -> pure $ Boolean b- A.Null -> pure $ Keyword (Ident "null")- A.Object hm -> do- let kvs = HashMap.toList hm- vs <- traverse parseJSON (snd <$> kvs)- pure . Dict . Map.fromList $ zip (String . fst <$> kvs) vs--- | Convert a radicle `Value` into an 'aeson' value, if possible.---- >>> import Data.Aeson (encode)diff --git a/src/Radicle/Internal/MachineBackend/Ipfs.hs b/src/Radicle/Internal/MachineBackend/Ipfs.hsindex d6a3414..a301841 100644--- a/src/Radicle/Internal/MachineBackend/Ipfs.hs+++ b/src/Radicle/Internal/MachineBackend/Ipfs.hs@@ -15,10 +15,7 @@-- }-- @---module Radicle.Internal.MachineBackend.Ipfs- ( ipfsPrimFns- , ipfsMachineCreate- ) where+module Radicle.Internal.MachineBackend.Ipfs whereimport Protolude hiding (TypeError, catch, try)diff --git a/src/Radicle/Ipfs.hs b/src/Radicle/Ipfs.hsindex 0402037..de1a5fe 100644--- a/src/Radicle/Ipfs.hs+++ b/src/Radicle/Ipfs.hs@@ -13,26 +13,38 @@ module Radicle.Ipfs, DagPutResponse(..), dagPut-, dagGet, namePublish, NameResolveResponse(..), nameResolve++ , PubsubMessage(..)+ , publish+ , subscribe) whereimport Protolude hiding (TypeError, catch, try)import Control.Exception.Safeimport Control.Monad.Fail+import Control.Monad.Trans.Resourceimport Data.Aeson (FromJSON, ToJSON, (.:), (.=))import qualified Data.Aeson as Aesonimport qualified Data.Aeson.Types as Aeson+import qualified Data.ByteString.Multibase as Multibase+import Data.Conduit ((.|))+import qualified Data.Conduit as C+import qualified Data.Conduit.Attoparsec as C+import qualified Data.Conduit.Combinators as Cimport Data.IPLD.CIDimport qualified Data.Text as T+import qualified Data.Text.Encoding as Timport Lens.Micro ((.~), (^.))import Network.HTTP.Client(HttpException(..), HttpExceptionContent(..))+import qualified Network.HTTP.Client as HTTP+import qualified Network.HTTP.Conduit as HTTPimport qualified Network.Wreq as Wreqimport System.Environment (lookupEnv)@@ -103,6 +115,55 @@ addressFromText t =-- * IPFS node API--------------------------------------------------------------------------+data PubsubMessage = PubsubMessage+ { messageTopicIDs :: [Text]+ , messageData :: ByteString+ , messageFrom :: ByteString+ , messageSeqno :: ByteString+ } deriving (Eq, Show)++instance FromJSON PubsubMessage where+ parseJSON = Aeson.withObject "PubsubMessage" $ \o -> do+ messageTopicIDs <- o .: "topicIDs"+ Right messageData <- o .: "data" <&> T.encodeUtf8 <&> Multibase.decodeBase64+ Right messageFrom <- o .: "from" <&> T.encodeUtf8 <&> Multibase.decodeBase64+ Right messageSeqno <- o .: "seqno" <&> T.encodeUtf8 <&> Multibase.decodeBase64+ pure PubsubMessage {..}++-- | Subscribe to a topic and call @messageHandler@ on every message.+-- The IO action blocks while we are subscribed. To stop subscription+-- you need to kill the thread the subscription is running in.+subscribe :: Text -> (PubsubMessage -> IO ()) -> IO ()+subscribe topic messageHandler = runResourceT $ do+ mgr <- liftIO $ HTTP.newManager HTTP.defaultManagerSettings+ req <- HTTP.parseRequest "http://localhost:9301/api/v0/pubsub/sub" <&>+ HTTP.setQueryString+ [ ("arg", Just $ T.encodeUtf8 topic)+ , ("encoding", Just "json")+ , ("stream-channels", Just "true")+ ]+ body <- HTTP.responseBody <$> HTTP.http req mgr+ C.runConduit $ body .| fromJSONC .| C.mapM_ (liftIO . messageHandler) .| C.sinkNull+ pure ()+ where+ fromJSONC :: (MonadThrow m, Aeson.FromJSON a) => C.ConduitT ByteString a m ()+ fromJSONC = jsonC .| C.mapM parseThrow++ jsonC :: (MonadThrow m) => C.ConduitT ByteString Aeson.Value m ()+ jsonC = C.peekForever (C.sinkParser Aeson.json >>= C.yield)++ parseThrow :: (MonadThrow m, Aeson.FromJSON a) => Aeson.Value -> m a+ parseThrow value = do+ case Aeson.fromJSON value of+ Aeson.Error err -> throwString err+ Aeson.Success a -> pure a+++-- | Publish a message to a topic.+publish :: Text -> LByteString -> IO ()+publish topic message =+ void $ ipfsHttpPost' "pubsub/pub" [("arg", topic)] "data" message+newtype KeyGenResponse = KeyGenResponse IpnsIdkeyGen :: Text -> IO KeyGenResponse@@ -180,12 +241,21 @@ ipfsHttpPost-> LByteString -- ^ Payload argument-> IO aipfsHttpPost path params payloadArgName payload = do- let opts = Wreq.defaults & Wreq.params .~ params- url <- ipfsApiUrl path- res <- Wreq.postWith opts (toS url) (Wreq.partLBS payloadArgName payload) `catch` handleRequestException+ res <- ipfsHttpPost' path params payloadArgName payloadjsonRes <- Wreq.asJSON res `catch` handleParseException pathpure $ jsonRes ^. Wreq.responseBody+ipfsHttpPost'+ :: Text -- ^ Path of the endpoint under "/api/v0/"+ -> [(Text, Text)] -- ^ URL query parameters+ -> Text -- ^ Name of the argument for payload+ -> LByteString -- ^ Payload argument+ -> IO (Wreq.Response LByteString)+ipfsHttpPost' path params payloadArgName payload = do+ let opts = Wreq.defaults & Wreq.params .~ params+ url <- ipfsApiUrl path+ Wreq.postWith opts (toS url) (Wreq.partLBS payloadArgName payload) `catch` handleRequestException+ipfsApiUrl :: Text -> IO TextipfsApiUrl path = dobaseUrl <- fromMaybe "http://localhost:9301" <$> lookupEnv "RAD_IPFS_API_URL"