Skip to content

A peer-to-peer stack for code collaboration

Implement Radicle Daemon

James Henri Haydonf42ed2ef1e26
diff --git a/.stylish-haskell.yaml b/.stylish-haskell.yaml
index f157e71..f791b62 100644
--- a/.stylish-haskell.yaml
+++ b/.stylish-haskell.yaml
@@ -203,4 +203,6 @@ language_extensions:
- TupleSections
- TypeFamilies
- TypeOperators
+ - NamedFieldPuns
- TypeApplications
+ - ViewPatterns
diff --git a/.weeder.yaml b/.weeder.yaml
index f883d5c..f07fedd 100644
--- a/.weeder.yaml
+++ b/.weeder.yaml
@@ -2,11 +2,6 @@
- name: radicle
- section:
- name: test:spec
- - message:
- - name: Redundant build-depends entry
- - depends:
- # Required as compiler plugin
- - tasty-discover
- message:
# BEGIN export all for ease of debugging
- name: Weeds exported
diff --git a/bin/rad-ipfs-daemon b/bin/rad-ipfs-daemon
index 01ccea8..34ec828 100755
--- a/bin/rad-ipfs-daemon
+++ b/bin/rad-ipfs-daemon
@@ -24,4 +24,4 @@ ipfs config Addresses.API "/ip4/127.0.0.1/tcp/9301"
ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/9302"
ipfs config --json Addresses.Swarm '["/ip4/0.0.0.0/tcp/9303", "/ip6/::/tcp/9303"]'
-ipfs daemon "$@"
+ipfs daemon --enable-pubsub-experiment "$@"
diff --git a/exe/Daemon.hs b/exe/Daemon.hs
new file mode 100644
index 0000000..4279149
--- /dev/null
+++ b/exe/Daemon.hs
@@ -0,0 +1,583 @@
+-- | The radicle-daemon; a long-running background process which
+-- materialises the state of remote IPFS machines on the user's PC, and
+-- writes to those IPFS machines the user is an owner of.
+--
+-- See
+-- <https://github.com/oscoin/radicle/blob/master/rfcs/0003-radicle-daemon.rst
+-- the RFC>.
+module Daemon where
+
+import Protolude hiding (fromStrict, option, poll)
+
+import qualified Data.Aeson as A
+import Data.ByteString.Lazy (fromStrict)
+import qualified Data.Map.Strict as Map
+import qualified Data.Sequence as Seq
+import qualified Data.Text as T
+import qualified Data.Time.Clock.System as Time
+import Network.Wai.Handler.Warp (run)
+import Options.Applicative
+import Servant
+import System.Directory (doesFileExist)
+
+import Radicle.Daemon.Common hiding (logInfo)
+import qualified Radicle.Daemon.Common as Common
+import qualified Radicle.Internal.UUID as UUID
+
+import Radicle hiding (Env)
+import Radicle.Daemon.HttpApi
+import Radicle.Daemon.Ipfs
+import qualified Radicle.Internal.CLI as Local
+import qualified Radicle.Internal.ConcurrentMap as CMap
+
+
+-- TODO(james): Check that the IPFS functions are doing all the
+-- necessary pinning.
+
+-- * Types
+
+data MachineError
+ = InvalidInput (LangError Value)
+ | IpfsError IpfsError
+ | AckTimeout
+ | DaemonError Text
+ | MachineAlreadyCached
+ | MachineNotCached
+
+data Error
+ = MachineError MachineId MachineError
+ | CouldNotCreateMachine Text
+
+displayError :: Error -> (Text, [(Text,Text)])
+displayError = \case
+ CouldNotCreateMachine e -> ("Could not create IPFS machine", [("error", e)])
+ MachineError id e -> let mid = ("machine-id", getMachineId id) in
+ case e of
+ InvalidInput err -> ("Invalid radicle input", [mid, ("error", renderCompactPretty err)])
+ DaemonError err -> ("Internal error", [mid, ("error", err)])
+ IpfsError e' -> case e' of
+ IpfsDaemonError err -> ("There was an error using IPFS", [mid, ("error", toS (displayException err))])
+ InternalError err -> ("Internal error", [mid, ("error", err)])
+ NetworkError err -> ("There was an error communicating with the IPFS daemon", [mid, ("error", err)])
+ AckTimeout -> ("The writer appears to be offline", [mid])
+ MachineAlreadyCached -> ("Tried to add already cached machine", [mid])
+ MachineNotCached -> ("Machine was not found in cache", [mid])
+
+logDaemonError :: MonadIO m => Error -> m ()
+logDaemonError (displayError -> (m, xs)) = logErr m xs
+
+type Follows = Map MachineId ReaderOrWriter
+
+-- * Main
+
+data Opts = Opts
+ { port :: Int
+ -- TODO(james): temporary, just for testing.
+ , filePrefix :: Text
+ , debug :: Bool
+ }
+
+opts :: Parser Opts
+opts = Opts
+ <$> option auto
+ ( long "port"
+ <> help "daemon port"
+ <> metavar "PORT"
+ <> showDefault
+ <> value 8909
+ )
+ <*> strOption
+ ( long "filePrefix"
+ <> help "file prefix"
+ <> metavar "PREFIX"
+ <> showDefault
+ <> value ""
+ )
+ <*> switch
+ ( long "debug"
+ <> help "enable debug logging"
+ <> showDefault
+ )
+
+type FollowFileLock = MVar ()
+
+data LogLevel = Normal | Debug
+ deriving (Eq, Ord)
+
+data Env = Env
+ { followFileLock :: FollowFileLock
+ , followFile :: FilePath
+ , machines :: IpfsMachines
+ , logLevel :: LogLevel
+ }
+
+newtype Daemon a = Daemon { fromDaemon :: ExceptT Error (ReaderT Env IO) a }
+ deriving (Functor, Applicative, Monad, MonadError Error, MonadIO, MonadReader Env)
+
+runDaemon :: Env -> Daemon a -> IO (Either Error a)
+runDaemon env (Daemon x) = runReaderT (runExceptT x) env
+
+main :: IO ()
+main = do
+ Opts{..} <- execParser allOpts
+ followFileLock <- newMVar ()
+ followFile <- Local.getRadicleFile (toS filePrefix <> "daemon-follows")
+ machines <- Chains <$> CMap.empty
+ let env = Env{ logLevel = if debug then Debug else Normal, ..}
+ follows <- readFollowFileIO followFileLock followFile
+ initRes <- runDaemon env (init follows)
+ case initRes of
+ Left err -> logErr "Init failed" [] >> logDaemonError err
+ Right _ -> do
+ polling <- async $ initPolling env
+ let app = serve daemonApi (server env)
+ Common.logInfo "Start listening" [("port", show port)]
+ serv <- async $ run port app
+ exc <- waitEitherCatchCancel polling serv
+ case exc of
+ Left (Left err) -> logErr "Polling failed with an exception" [("error", toS (displayException err))]
+ Left (Right void') -> absurd void'
+ Right (Left err) -> logErr "Server failed with an exception" [("error", toS (displayException err))]
+ Right (Right ()) -> logErr "Server stopped (this should not happen)" []
+ where
+ allOpts = info (opts <**> helper)
+ ( fullDesc
+ <> progDesc "Run the radicle daemon"
+ <> header "radicle-daemon"
+ )
+
+type IpfsMachine = Chain MachineId MachineEntryIndex TopicSubscription
+type IpfsMachines = Chains MachineId MachineEntryIndex TopicSubscription
+
+server :: Env -> Server DaemonApi
+server env = hoistServer daemonApi nt daemonServer
+ where
+ daemonServer :: ServerT DaemonApi Daemon
+ daemonServer = (machineEndpoints :<|> newMachine) :<|> pure swagger
+ machineEndpoints id = query id :<|> send id
+
+ nt :: Daemon a -> Handler a
+ nt d = do
+ x_ <- liftIO $ runDaemon env d
+ case x_ of
+ Left err -> throwError (toServantErr err)
+ Right x -> pure x
+
+ toServantErr err@(MachineError _ e) = case e of
+ InvalidInput e' -> err400 { errBody = fromStrict $ encodeUtf8 (renderPrettyDef e') }
+ AckTimeout -> err504 { errBody = "The writer for this IPFS machine does not appear to be online." }
+ _ -> case displayError err of
+ (msg, _) -> err500 { errBody = toS msg }
+ toServantErr (CouldNotCreateMachine err) = err500 { errBody = "Could not create IPFS machine: " <> toS err }
+
+-- * Init
+
+-- | Initiate machines according to follow file.
+init :: Follows -> Daemon ()
+init follows = traverse_ initMachine (Map.toList follows)
+ where
+ initMachine (id, Reader) = initAsReader id
+ initMachine (id, Writer) = initAsWriter id
+
+-- * Endpoints
+
+-- | Create a new IPFS machine and initialise the daemon as as the
+-- /writer/.
+newMachine :: Daemon NewResponse
+newMachine = do
+ id <- create
+ sub <- ipfs id $ initSubscription id
+ logInfo Normal "Created new IPFS machine" [("machine-id", getMachineId id)]
+ m <- liftIO $ emptyMachine id Writer sub
+ insertNewMachine m
+ actAsWriter m
+ writeFollowFile
+ pure (NewResponse id)
+ where
+ create = Daemon $ mapExceptT (lift . fmap (first CouldNotCreateMachine)) $ createMachine
+
+-- | Evaluate an expression against a cached machine. The resulting
+-- state is always discarded, and the expression is never sent to the
+-- writer. Only used for local queries.
+--
+-- Hitting this endpoint will turn on high-frequency polling for a
+-- fixed amount of time.
+query :: MachineId -> QueryRequest -> Daemon QueryResponse
+query id (QueryRequest v) = do
+ m <- checkMachineLoaded id
+ bumpPolling id
+ case fst <$> runIdentity $ runLang (chainState m) $ eval v of
+ Left err -> throwError $ MachineError id (InvalidInput err)
+ Right rv -> do
+ logInfo Normal "Query success:" [("result", renderCompactPretty rv)]
+ pure (QueryResponse rv)
+
+-- | Write a new expression to an IPFS machine.
+--
+-- - If the daemon is the writer for the machine, it will write the
+-- new inputs to IPFS and then send out a notification on the
+-- machine's pubsub topic.
+--
+-- - If the daemon is a reader for the machine, it will request the
+-- machine's writer daemon to perform the write, and wait for an
+-- ack.
+send :: MachineId -> SendRequest -> Daemon SendResponse
+send id (SendRequest expressions) = do
+ mode_ <- machineMode
+ case mode_ of
+ Just (Writer, _) -> do
+ results <- writeInputs id expressions Nothing
+ pure SendResponse{..}
+ Just (Reader, sub) -> requestInput sub
+ Nothing -> do
+ m <- initAsReader id
+ requestInput (chainSubscription m)
+ where
+ requestInput sub = do
+ nonce <- liftIO $ UUID.uuid
+ let isResponse = \case
+ New InputsApplied{ nonce = Just nonce' } | nonce' == nonce -> True
+ _ -> False
+ asyncMsg <- liftIO $ async $ subscribeOne sub ackWaitTime isResponse (logNonDecodableMsg id)
+ ipfs id $ publish id (Submit SubmitInputs{..})
+ logInfo Debug
+ "Sent input request to writer"
+ [ ("machine-id", getMachineId id)
+ , ("expressions", prValues expressions) ]
+ msg_ <- liftIO $ waitCatch asyncMsg
+ case msg_ of
+ Right (Just (New InputsApplied{results})) -> do
+ logInfo Normal
+ "Writer accepted input request"
+ [ ("machine-id", getMachineId id)
+ , ("results", prValues results)
+ ]
+ bumpPolling id
+ pure SendResponse{..}
+ Right Nothing -> throwError $ MachineError id AckTimeout
+ Right _ -> throwError $ MachineError id (DaemonError "Didn't filter machine topic messages correctly.")
+ Left err -> throwError $ MachineError id (DaemonError (toS (displayException err)))
+
+ machineMode = fmap (liftA2 (,) chainMode chainSubscription) <$> lookupMachine id
+ prValues = T.intercalate "," . (renderCompactPretty <$>)
+
+-- * Helpers
+
+logInfo :: LogLevel -> Text -> [(Text,Text)] -> Daemon ()
+logInfo l msg infos = do
+ l' <- asks logLevel
+ if l <= l'
+ then Common.logInfo msg infos
+ else pure ()
+
+withFollowFileLock :: FollowFileLock -> IO a -> IO a
+withFollowFileLock lock = withMVar lock . const
+
+readFollowFileIO :: FollowFileLock -> FilePath -> IO Follows
+readFollowFileIO lock ff = withFollowFileLock lock $ do
+ exists <- doesFileExist ff
+ t <- if exists
+ then readFile ff
+ else let noFollows = toS (A.encode (Map.empty :: Follows))
+ in writeFile ff noFollows $> noFollows
+ case A.decode (toS t) of
+ Nothing -> panic $ "Invalid daemon-follow file: could not decode " <> toS ff
+ Just fs -> pure fs
+
+writeFollowFile :: Daemon ()
+writeFollowFile = do
+ lock <- asks followFileLock
+ Chains cMap <- asks machines
+ ff <- asks followFile
+ liftIO $ withFollowFileLock lock $ do
+ ms <- CMap.nonAtomicRead cMap
+ let fs = Map.fromList $ second chainMode <$> Map.toList ms
+ writeFile ff (toS $ A.encode fs)
+
+lookupMachine :: MachineId -> Daemon (Maybe IpfsMachine)
+lookupMachine id = do
+ msVar <- asks machines
+ liftIO $ CMap.lookup id (getChains msVar)
+
+-- | Given an 'MachineId', makes sure the machine is in the cache and
+-- updated.
+checkMachineLoaded :: MachineId -> Daemon IpfsMachine
+checkMachineLoaded id = do
+ m_ <- lookupMachine id
+ case m_ of
+ Nothing -> do
+ -- In this case we have not seen the machine before so we act as
+ -- a reader.
+ m <- initAsReader id
+ writeFollowFile
+ pure m
+ Just m -> case chainMode m of
+ Writer -> pure m
+ Reader -> refreshAsReader id
+ -- TODO(james): For the moment we will just force a
+ -- refresh. Later consider brining back the check to see if
+ -- the machine was very recently updated.
+
+ -- do
+ -- delta <- liftIO $ sinceLastUpdate m
+ -- -- If machine is half a second fresh, then return it.
+ -- if delta < 500
+ -- then pure m
+ -- else refreshAsReader id
+
+logNonDecodableMsg :: MachineId -> Text -> IO ()
+logNonDecodableMsg (MachineId id) bad =
+ Common.logInfo "Non-decodable message on machine's pubsub topic" [("machine-id", id), ("message", bad)]
+
+daemonHandler :: Env -> MachineId -> (Message -> Daemon ()) -> Either Text Message -> IO ()
+daemonHandler _ id _ (Left bad) = logNonDecodableMsg id bad
+daemonHandler env _ h (Right msg) = do
+ x_ <- runDaemon env (h msg)
+ case x_ of
+ Left err -> logDaemonError err
+ Right _ -> pure ()
+
+-- | Loads a machine fresh from IPFS.
+loadMachine :: ReaderOrWriter -> MachineId -> Daemon IpfsMachine
+loadMachine mode id = do
+ (idx, is) <- ipfs id $ machineInputsFrom id Nothing
+ sub <- ipfs id $ initSubscription id
+ m <- liftIO $ emptyMachine id mode sub
+ (m', _) <- addInputs is (pure idx) (const (pure ())) m
+ insertNewMachine m'
+ pure m'
+
+-- | Add inputs to a cached machine.
+addInputs
+ :: [Value]
+ -- ^ Inputs to add.
+ -> Daemon MachineEntryIndex
+ -- ^ Determine the new index.
+ -> ([Value] -> Daemon ())
+ -- ^ Performed after the chain is cached.
+ -> IpfsMachine
+ -> Daemon (IpfsMachine, ([Value], MachineEntryIndex))
+ -- ^ Returns the updated machine, the results and the new index.
+addInputs is getIdx after m =
+ case advanceChain m is of
+ Left err -> throwError $ MachineError (chainName m) (InvalidInput err)
+ Right (rs, newState) -> do
+ idx <- getIdx
+ t <- liftIO $ Time.getSystemTime
+ let news = Seq.fromList $ zip is rs
+ m' = m { chainState = newState
+ , chainLastIndex = Just idx
+ , chainEvalPairs = chainEvalPairs m Seq.>< news
+ , chainLastUpdated = t
+ }
+ after rs
+ pure (m', (rs, idx))
+
+ipfs :: MachineId -> ExceptT IpfsError IO a -> Daemon a
+ipfs id = Daemon . mapExceptT (lift . fmap (first (MachineError id . IpfsError)))
+
+-- | Do some high-freq polling for a while.
+bumpPolling :: MachineId -> Daemon ()
+bumpPolling id = do
+ modifyMachine id $ \m' -> pure (m' { chainPolling = highFreq }, () )
+ logInfo Debug "Reset to high-frequency polling" [("machine-id", getMachineId id)]
+
+-- | Insert a new machine into the cache. Errors if the machine is
+-- already cached.
+insertNewMachine :: IpfsMachine -> Daemon ()
+insertNewMachine m = do
+ msVar <- asks machines
+ inserted_ <- liftIO $ CMap.insertNew id m (getChains msVar)
+ case inserted_ of
+ Just () -> pure ()
+ Nothing -> throwError (MachineError id MachineAlreadyCached)
+ where
+ id = chainName m
+
+-- | Modify a machine that is already in the cache. Errors if the
+-- machine isn't in the cache already.
+modifyMachine :: MachineId -> (IpfsMachine -> Daemon (IpfsMachine, a)) -> Daemon a
+modifyMachine id f = do
+ env <- ask
+ res <- liftIO $ CMap.modifyExistingValue id (getChains (machines env)) $ \m -> do
+ x <- runDaemon env (f m)
+ pure $ case x of
+ Left err -> (m, Left err)
+ Right (m', y) -> (m', Right y)
+ case res of
+ Nothing -> throwError (MachineError id MachineNotCached)
+ Just (Left err) -> throwError err
+ Just (Right y) -> pure y
+
+emptyMachine :: MachineId -> ReaderOrWriter -> TopicSubscription -> IO IpfsMachine
+emptyMachine id mode sub = do
+ t <- Time.getSystemTime
+ pure Chain{ chainName = id
+ , chainState = pureEnv
+ , chainEvalPairs = mempty
+ , chainLastIndex = Nothing
+ , chainMode = mode
+ , chainSubscription = sub
+ , chainLastUpdated = t
+ , chainPolling = highFreq
+ }
+
+-- ** Reader
+
+-- | Load a machine in /reader mode/ and return it.
+--
+-- Loads the machines input log from IPFS and listens for machine
+-- updates on pubsub.
+--
+-- The machine is added to the deamons machine cache.
+initAsReader :: MachineId -> Daemon IpfsMachine
+initAsReader id = do
+ m <- loadMachine Reader id
+ env <- ask
+ _ <- liftIO $ addHandler (chainSubscription m) (daemonHandler env id onMsg)
+ logInfo Normal "Following as reader" [ ("machine-id", getMachineId id)
+ , ("current-input-index", show (chainLastIndex m)) ]
+ pure m
+ where
+ onMsg :: Message -> Daemon ()
+ onMsg = \case
+ New InputsApplied{..} -> do
+ _ <- refreshAsReader id
+ -- TODO(james): decide if this should bump polling. If this is
+ -- a very active chain this will mean the polling is always
+ -- high-frequency.
+ bumpPolling id
+ _ -> pure ()
+
+-- | Updates a machine in the cache with the new inputs pulled from IPFS.
+refreshAsReader :: MachineId -> Daemon IpfsMachine
+refreshAsReader id = modifyMachine id $ \m -> do
+ let currentIdx = chainLastIndex m
+ (newIdx, is) <- ipfs id $ machineInputsFrom (chainName m) currentIdx
+ if currentIdx == Just newIdx
+ then do
+ logInfo Debug
+ "Reader is already up to date"
+ [mid, ("input-index", show newIdx)]
+ pure (m, m)
+ else do
+ (m', _) <- addInputs is (pure newIdx) (const (pure ())) m
+ logInfo Debug
+ "Updated reader"
+ [mid, ("n", show (length is)), ("input-index", show newIdx)]
+ pure (m', m')
+ where
+ mid = ("machine-id", getMachineId id)
+
+-- ** Writer
+
+initAsWriter :: MachineId -> Daemon IpfsMachine
+initAsWriter id = do
+ m <- loadMachine Writer id
+ actAsWriter m
+ pure m
+
+-- | Subscribes the daemon to the machine's pubsub topic to listen for
+-- input requests.
+actAsWriter :: IpfsMachine -> Daemon ()
+actAsWriter m = do
+ env <- ask
+ _ <- liftIO $ addHandler (chainSubscription m) (daemonHandler env id onMsg)
+ logInfo Normal "Acting as writer" [("machine-id", getMachineId id)]
+ where
+ id = chainName m
+ onMsg = \case
+ Submit SubmitInputs{..} -> writeInputs id expressions (Just nonce) >> pure ()
+ _ -> pure ()
+
+-- | Write and evaluate inputs in a machine we control.
+--
+-- Returns the outputs generated by evaluating the inputs.
+--
+-- Sends out a 'InputsApplied' message over pubsub that includes the
+-- outputs and the given nonce.
+writeInputs :: MachineId -> [Value] -> Maybe Text -> Daemon [Value]
+writeInputs id is nonce = do
+ (rs, idx) <- modifyMachine id $ addInputs is write pub
+ logInfo Debug "Wrote inputs to IPFS" [ ("machine-id", getMachineId id)
+ , ("new-input-index", show idx) ]
+ pure rs
+ where
+ write = ipfs id $ writeIpfs id is
+ pub results = ipfs id $ publish id (New InputsApplied{..})
+
+-- * Polling
+
+-- | Fetch and apply new inputs for all machines in reader mode.
+poll :: Daemon ()
+poll = do
+ msVar <- asks machines
+ ms <- liftIO $ CMap.nonAtomicRead (getChains msVar)
+ traverse_ pollMachine ms
+ where
+ pollMachine :: IpfsMachine -> Daemon ()
+ pollMachine m@Chain{chainMode = Reader, ..} = do
+ delta <- liftIO $ sinceLastUpdate m
+ let (shouldPoll, newPoll) =
+ case chainPolling of
+ HighFreq more ->
+ let more' = more - delta
+ in if more' > 0
+ then (True, HighFreq more')
+ else (False, LowFreq)
+ -- Low frequency polling is every 10 seconds.
+ LowFreq -> (delta > lowFreqPollPeriod, LowFreq)
+ when shouldPoll $ do
+ logInfo Debug "Polling.." [("machine-id", getMachineId chainName)]
+ refreshAsReader chainName >> pure ()
+ modifyMachine chainName $ \m' -> pure (m' { chainPolling = newPoll }, () )
+ pollMachine _ = pure ()
+
+-- | Returns the amount of time since the last time the machine was
+-- updated.
+sinceLastUpdate :: IpfsMachine -> IO Milliseconds
+sinceLastUpdate m = timeDelta (chainLastUpdated m) <$> Time.getSystemTime
+ where
+ timeDelta x y =
+ case (trunc x, trunc y) of
+ (Time.MkSystemTime s n, Time.MkSystemTime s' n') -> 1000 * (s' - s) + fromIntegral (n' - n) `div` 1000000
+ trunc = Time.truncateSystemTimeLeapSecond
+
+-- | Polling loop that looks for changes on all loaded reader machines
+-- and updates them.
+initPolling :: Env -> IO Void
+initPolling env = do
+ threadDelay (millisToMicros highFreqPollPeriod)
+ res <- runDaemon env poll
+ -- If polling encounters an error it should log it but continue.
+ -- Later we might detect some errors as critical and halt the
+ -- daemon.
+ case res of
+ Left err -> logDaemonError err
+ Right _ -> pure ()
+ initPolling env
+
+-- * Timings
+
+type Milliseconds = Int64
+
+millisToMicros :: Milliseconds -> Int
+millisToMicros n = 1000 * fromIntegral n
+
+-- | High-frequency polling happens once every half-second.
+highFreqPollPeriod :: Milliseconds
+highFreqPollPeriod = 500
+
+-- | Low-frequency polling happens once every 10 seconds.
+lowFreqPollPeriod :: Milliseconds
+lowFreqPollPeriod = 10 * 1000
+
+-- | The amount of time a reader will wait for a response message from
+-- a writer: 8 seconds.
+ackWaitTime :: Milliseconds
+ackWaitTime = 8 * 1000
+
+-- | The amount of time a machine does high-frequency polling for
+-- before it returns to low-frequency polling: 10 minutes.
+highFreq :: Polling
+highFreq = HighFreq (10 * 60 * 1000)
diff --git a/exe/ReferenceDoc.hs b/exe/ReferenceDoc.hs
index 5b7d7cc..73dd0cc 100644
--- a/exe/ReferenceDoc.hs
+++ b/exe/ReferenceDoc.hs
@@ -13,6 +13,7 @@ import qualified Data.Map.Strict as Map
import qualified Data.Text as T
import Data.Yaml hiding (Value)
import Radicle
+import qualified Radicle.Daemon.HttpApi as Daemon
import Radicle.Internal.Core
import qualified Radicle.Internal.Doc as Doc
import Radicle.Internal.Identifier
@@ -43,6 +44,7 @@ main = do
let env = bindingsEnv $ bindingsFromRadicle res `lPanic` "Couldn't convert radicle state."
let rst = runPure (writeRST Default.def $ Pandoc nullMeta (doc content env)) `lPanic` "Couldn't generate RST"
writeFile "docs/source/reference.rst" rst
+ writeFile "docs/source/daemon-api.yaml" (toS (encode Daemon.swagger))
where
doc :: Content -> Env Value -> [Block]
diff --git a/exe/Server.hs b/exe/Server.hs
index 26f373a..5e8aa16 100644
--- a/exe/Server.hs
+++ b/exe/Server.hs
@@ -8,7 +8,7 @@ import qualified Data.Aeson as A
import Data.ByteString.Lazy (fromStrict)
import qualified Data.Map as Map
import qualified Data.Sequence as Seq
-import qualified Data.Text as T
+import qualified Data.Time.Clock.System as Time
import Database.PostgreSQL.Simple
(ConnectInfo(..), Connection, connect)
import Network.Wai.Handler.Warp (run)
@@ -48,6 +48,8 @@ import Servant
)
import Radicle
+import Radicle.Daemon.Common
+import qualified Radicle.Internal.ConcurrentMap as CMap
import Radicle.Internal.MachineBackend.EvalServer
import Server.DB
@@ -56,16 +58,6 @@ import Server.DB
newtype Exprs = Exprs { getExprs :: Seq Value }
deriving (Generic, Eq, Ord)
-data Chain = Chain
- { chainName :: Text
- , chainState :: Bindings (PrimFns Identity)
- , chainEvalPairs :: Seq.Seq (Value, Value)
- } deriving (Generic)
-
--- For efficiency we might want keys to also be mvars, but efficiency doesn't
--- matter for a test server.
-newtype Chains = Chains { getChains :: MVar (Map Text Chain) }
-
type ServerApi
= "chains" :> Capture "chain" Text :> (ChainSubmitEndpoint :<|> ChainSinceEndpoint)
:<|> "outputs" :> Capture "chain" Text :> Get '[JSON] [Maybe A.Value]
@@ -74,38 +66,39 @@ type ServerApi
serverApi :: Proxy ServerApi
serverApi = Proxy
+type HttpChains = Chains Text Int ()
-- * Helpers
-- | Returns the ID of the last inserted expression which is the chain
-- length minus one.
-insertExpr :: Connection -> Chains -> Text -> [Value] -> IO (Either Text Int)
-insertExpr conn chains name vals = modifyMVar (getChains chains) $ \c -> do
- let maybeChain = Map.lookup name c
+insertExpr :: Connection -> HttpChains -> Text -> [Value] -> IO (Either Text Int)
+insertExpr conn chains name vals = CMap.modifyValue name (getChains chains) $ \maybeChain -> do
chain <- case maybeChain of
Nothing -> do
logInfo "Creating new machine" [("machine", name)]
- pure $ Chain name pureEnv mempty
+ t <- Time.getSystemTime
+ pure $ Chain name pureEnv mempty Nothing Writer () t LowFreq
Just chain' -> pure chain'
- let (r :: Either (LangError Value) [Value], newSt) = runIdentity
- $ runLang (chainState chain)
- $ traverse eval vals
- case r of
+ case advanceChain chain vals of
Left e -> do
logInfo "Submitted expressions failed to evaluate" [("machine", name)]
- pure . (c ,) . Left $ renderPrettyDef e
- Right valsRes -> do
+ pure (Nothing, Left $ renderPrettyDef e)
+ Right (valsRes, newSt) -> do
traverse_ (insertExprDB conn name) vals
let news = Seq.fromList $ zip vals valsRes
+ let pairs = chainEvalPairs chain Seq.>< news
+ let lastIndex = Seq.length pairs - 1
let chain' = chain { chainState = newSt
- , chainEvalPairs = chainEvalPairs chain Seq.>< news
+ , chainEvalPairs = pairs
+ , chainLastIndex = Just lastIndex
}
- pure (Map.insert name chain' c, Right $ Seq.length (chainEvalPairs chain') - 1)
+ pure (Just chain', Right lastIndex)
-- | Load the state from the DB, returning an MVar with resulting state
-- and values.
-loadState :: Connection -> IO Chains
+loadState :: Connection -> IO HttpChains
loadState conn = do
createIfNotExists conn
res <- getAllDB conn
@@ -117,12 +110,19 @@ loadState conn = do
logInfo "Failed to load machine" [("machine", name)]
panic $ show err
(Right pairs, st') -> do
+ t <- Time.getSystemTime
let c = Chain { chainName = name
, chainState = st'
, chainEvalPairs = pairs
+ , chainLastIndex = Just $ Seq.length pairs - 1
+ , chainMode = Writer
+ , chainSubscription = ()
+ , chainLastUpdated = t
+ , chainPolling = LowFreq
}
pure (name, c)
- chains' <- newMVar $ Map.fromList chainPairs
+ --chains' <- newMVar $ Map.fromList chainPairs
+ chains' <- CMap.fromMap $ Map.fromList chainPairs
logInfo "Loaded machines into memory" [("machine-count", show $ length chainPairs)]
pure $ Chains chains'
@@ -134,7 +134,7 @@ loadState conn = do
-- second the expressions being submitted. The server either accepts all or
-- rejects all expressions (though other systems may not provide this
-- guarantee).
-submit :: Connection -> Chains -> Text -> Values -> Handler Value
+submit :: Connection -> HttpChains -> Text -> Values -> Handler Value
submit conn chains name (Values vals) = do
logInfo "Submitted expressions" [("machine", name), ("expression-count", show $ length vals)]
res <- liftIO $ insertExpr conn chains name vals
@@ -149,31 +149,16 @@ since conn name index = do
logInfo "Requested expressions" [("machine", name), ("from-index", show index)]
liftIO $ Values <$> getSinceDB conn name index
-jsonOutputs :: Chains -> Text -> Handler [Maybe A.Value]
+jsonOutputs :: HttpChains -> Text -> Handler [Maybe A.Value]
jsonOutputs st name = do
- chains <- liftIO . readMVar $ getChains st
- pure $ case Map.lookup name chains of
+ maybeChain <- liftIO $ CMap.lookup name (getChains st)
+ pure $ case maybeChain of
Nothing -> []
Just chain -> toList $ maybeJson . snd <$> chainEvalPairs chain
static :: Server Raw
static = serveDirectoryFileServer "static/"
--- | Log a message to @stdout@.
---
--- The second argument is a list of key-value pairs that will be joined with "="
--- and appended to the message.
---
--- @
--- logInfo "the message" [("foo", "5)]
--- -- prints "INFO the message foo=5"
--- @
-logInfo :: MonadIO m => Text -> [(Text, Text)] -> m ()
-logInfo msg dat = do
- putStrLn $ "INFO " <> msg <> datString
- where
- datString = T.intercalate "" $ map (\(key, val) -> " " <> key <> "=" <> val) dat
-
-- * Opts
data Opts = Opts
@@ -247,7 +232,7 @@ main = do
<> header "radicle-server"
)
-server :: Connection -> Chains -> Server ServerApi
+server :: Connection -> HttpChains -> Server ServerApi
server conn chains = chainEndpoints :<|> jsonOutputs chains :<|> static
where
chainEndpoints chainName = submit conn chains chainName :<|> since conn chainName
diff --git a/package.yaml b/package.yaml
index 206fcc7..c1f5e28 100644
--- a/package.yaml
+++ b/package.yaml
@@ -29,6 +29,8 @@ library:
- aeson
- ansi-terminal
- bytestring
+ - conduit
+ - conduit-extra
- containers
- cryptonite
- directory
@@ -36,19 +38,24 @@ library:
- generics-eot
- haskeline
- http-client
+ - http-conduit
- ipld-cid
- megaparsec
- microlens
- mtl
+ - multibase
- pointed
- prettyprinter
- prettyprinter-ansi-terminal
- process
+ - resourcet
- safe-exceptions
- scientific
- serialise
- servant
- servant-client
+ - servant-swagger
+ - swagger2
- template-haskell
- text
- time
@@ -76,7 +83,6 @@ tests:
- serialise
- string-qq
- tasty
- - tasty-discover
- tasty-hunit
- tasty-quickcheck
- text
@@ -138,8 +144,27 @@ executables:
- warp
- wai-extra
- wai-cors
+ - time
ghc-options: -main-is Server
+ radicle-daemon:
+ main: Daemon.hs
+ source-dirs: exe
+ other-modules: []
+ dependencies:
+ - radicle
+ - aeson
+ - mtl
+ - containers
+ - bytestring
+ - directory
+ - optparse-applicative
+ - servant-server
+ - text
+ - warp
+ - time
+ ghc-options: -main-is Daemon
+
radicle-doc:
main: Doc.hs
source-dirs: exe/
@@ -205,6 +230,7 @@ default-extensions:
- DeriveFunctor
- DeriveGeneric
- DeriveTraversable
+- DuplicateRecordFields
- FlexibleContexts
- FlexibleInstances
- FunctionalDependencies
@@ -212,6 +238,7 @@ default-extensions:
- LambdaCase
- MultiParamTypeClasses
- MultiWayIf
+- NamedFieldPuns
- NoImplicitPrelude
- OverloadedStrings
- RankNTypes
@@ -221,3 +248,4 @@ default-extensions:
- TupleSections
- TypeFamilies
- TypeOperators
+- ViewPatterns
diff --git a/src/Radicle/Daemon/Common.hs b/src/Radicle/Daemon/Common.hs
new file mode 100644
index 0000000..7f6c8ff
--- /dev/null
+++ b/src/Radicle/Daemon/Common.hs
@@ -0,0 +1,66 @@
+module Radicle.Daemon.Common
+ ( ReaderOrWriter(..)
+ , Polling(..)
+ , Chain(..)
+ , Chains(..)
+ , logInfo
+ , logErr
+ , advanceChain
+ ) where
+
+import Protolude hiding (log)
+
+import qualified Data.Aeson as A
+import qualified Data.Sequence as Seq
+import qualified Data.Text as T
+import qualified Data.Time.Clock.System as Time
+
+import Radicle
+import qualified Radicle.Internal.ConcurrentMap as CMap
+
+data ReaderOrWriter = Reader | Writer
+ deriving (Generic)
+
+data Polling = HighFreq Int64 | LowFreq
+
+instance A.ToJSON ReaderOrWriter
+instance A.FromJSON ReaderOrWriter
+
+data Chain id idx subs = Chain
+ { chainName :: id
+ , chainState :: Bindings (PrimFns Identity)
+ , chainEvalPairs :: Seq.Seq (Value, Value)
+ , chainLastIndex :: Maybe idx
+ , chainMode :: ReaderOrWriter
+ , chainSubscription :: subs
+ , chainLastUpdated :: Time.SystemTime
+ , chainPolling :: Polling
+ } deriving (Generic)
+
+newtype Chains id idx subs = Chains { getChains :: CMap.CMap id (Chain id idx subs) }
+
+-- | Log a message to @stdout@.
+--
+-- The second argument is a list of key-value pairs that will be joined with "="
+-- and appended to the message.
+--
+-- @
+-- log "INFO" "the message" [("foo", "5)]
+-- -- prints "INFO the message foo=5"
+-- @
+log :: MonadIO m => Text -> Text -> [(Text, Text)] -> m ()
+log typ msg dat = do
+ putStrLn $ typ <> " " <> msg <> datString
+ where
+ datString = T.intercalate "" $ map (\(key, val) -> " " <> key <> "=" <> val) dat
+
+logInfo, logErr :: MonadIO m => Text -> [(Text, Text)] -> m ()
+logInfo = log "INFO "
+logErr = log "ERROR"
+
+advanceChain :: Chain id idx subs -> [Value] -> Either (LangError Value) ([Value], Bindings (PrimFns Identity))
+advanceChain chain vals =
+ let (r_, newSt) = runIdentity $ runLang (chainState chain) $ traverse eval vals
+ in case r_ of
+ Left e -> Left e
+ Right r -> pure (r, newSt)
diff --git a/src/Radicle/Daemon/HttpApi.hs b/src/Radicle/Daemon/HttpApi.hs
new file mode 100644
index 0000000..b5c1627
--- /dev/null
+++ b/src/Radicle/Daemon/HttpApi.hs
@@ -0,0 +1,123 @@
+{-# OPTIONS_GHC -fno-warn-orphans #-}
+module Radicle.Daemon.HttpApi
+ ( QueryRequest(..)
+ , QueryResponse(..)
+ , SendRequest(..)
+ , SendResponse(..)
+ , NewResponse(..)
+ , DaemonApi
+ , daemonApi
+ , swagger
+ ) where
+
+import qualified Prelude
+import Protolude
+
+import Data.Aeson ((.:))
+import qualified Data.Aeson as A
+import qualified Data.Aeson.Types as A
+import Data.Swagger
+import Data.Version (showVersion)
+import Radicle.Daemon.Ipfs
+import Servant.API
+import Servant.Swagger
+
+import qualified Paths_radicle as Radicle
+import Radicle
+
+instance FromHttpApiData MachineId where
+ parseUrlPiece = Right . MachineId . toS
+ parseQueryParam = parseUrlPiece
+
+singleKey :: Prelude.String -> (a -> b) -> (A.Object -> A.Parser a) -> A.Value -> A.Parser b
+singleKey tyName c k = A.withObject tyName $ k >=> pure . c
+
+newtype QueryRequest = QueryRequest { expression :: Value }
+ deriving (Generic)
+
+instance A.ToJSON QueryRequest where
+ toJSON = valueToJson . expression
+instance A.FromJSON QueryRequest where
+ parseJSON = singleKey "QueryRequest" QueryRequest $ (.: "expression") >=> jsonToValue
+
+newtype QueryResponse = QueryResponse { result :: Value }
+ deriving (Generic)
+
+instance A.ToJSON QueryResponse where
+ toJSON = valueToJson . result
+instance A.FromJSON QueryResponse where
+ parseJSON = singleKey "QueryResponse" QueryResponse $ (.: "result") >=> jsonToValue
+
+newtype SendRequest = SendRequest { expressions :: [Value] }
+ deriving (Generic)
+
+instance A.FromJSON SendRequest where
+ parseJSON = singleKey "SendRequest" SendRequest $ (.: "expressions") >=> traverse jsonToValue
+
+newtype SendResponse = SendResponse
+ { results :: [Value]
+ } deriving (Generic)
+
+instance A.ToJSON SendResponse where
+ toJSON = A.toJSON . fmap valueToJson . (results :: SendResponse -> [Value])
+
+newtype NewResponse = NewResponse
+ { machineId :: MachineId
+ } deriving (Generic)
+
+instance A.ToJSON NewResponse
+
+-- * APIs
+
+type Query = "query" :> ReqBody '[JSON] QueryRequest :> Post '[JSON] QueryResponse
+type Send = "send" :> ReqBody '[JSON] SendRequest :> Post '[JSON] SendResponse
+type New = "new" :> Post '[JSON] NewResponse
+
+type Machines = "machines" :> Capture "machineId" MachineId :> ( Query :<|> Send ) :<|> New
+type Docs = "docs" :> Get '[JSON] Swagger
+
+type DaemonApi = "v0" :> ( Machines :<|> Docs )
+
+daemonApi :: Proxy DaemonApi
+daemonApi = Proxy
+
+-- * Docs
+
+instance ToParamSchema MachineId where
+ toParamSchema _ = mempty { _paramSchemaType = SwaggerString }
+instance ToSchema MachineId where
+ declareNamedSchema pxy = pure $
+ NamedSchema (Just "MachineId") $
+ mempty { _schemaDescription = Just "The ID (i.e. IPNS name) of an IPFS machine."
+ , _schemaParamSchema = toParamSchema pxy }
+
+instance ToSchema Value where
+ declareNamedSchema _ = pure $
+ NamedSchema (Just "RadicleValue") $
+ mempty { _schemaDescription = Just "A radicle value formatted according to radicle's S-expression syntax."
+ , _schemaParamSchema = mempty {_paramSchemaType = SwaggerString } }
+
+instance ToSchema QueryRequest
+instance ToSchema QueryResponse
+instance ToSchema SendRequest
+instance ToSchema SendResponse
+instance ToSchema NewResponse
+
+instance ToSchema Swagger where
+ declareNamedSchema _ = pure $
+ NamedSchema (Just "Swagger") $
+ mempty { _schemaDescription = Just "This swagger spec."
+ , _schemaParamSchema = mempty { _paramSchemaType = SwaggerObject }}
+
+swagger :: Swagger
+swagger = swag { _swaggerInfo = inf }
+ where
+ inf = mempty { _infoTitle = "Radicle daemon"
+ , _infoDescription = Just desc
+ , _infoVersion = toS $ showVersion Radicle.version
+ }
+ swag = toSwagger daemonApi
+ desc =
+ "The radicle-daemon; a long-running background process which\
+ \ materialises the state of remote IPFS machines on the users\
+ \ PC, and writes to those IPFS machines the user is an owner of."
diff --git a/src/Radicle/Daemon/Ipfs.hs b/src/Radicle/Daemon/Ipfs.hs
new file mode 100644
index 0000000..e441814
--- /dev/null
+++ b/src/Radicle/Daemon/Ipfs.hs
@@ -0,0 +1,211 @@
+module Radicle.Daemon.Ipfs
+ ( IpfsError(..)
+ , MachineId(..)
+ , Message(..)
+ , InputsApplied(..)
+ , SubmitInputs(..)
+ , writeIpfs
+ , publish
+ , machineInputsFrom
+ , createMachine
+ , Ipfs.MachineEntryIndex
+ , TopicSubscription
+ , initSubscription
+ , subscribeOne
+ , addHandler
+ , valueToJson
+ , jsonToValue
+ ) where
+
+import Protolude hiding (bracket, catches)
+
+import Control.Exception.Safe
+import Control.Monad.Fail
+import Data.Aeson (decodeStrict, (.:), (.=))
+import qualified Data.Aeson as Aeson
+import qualified Data.Aeson.Types as Aeson
+import qualified Data.HashMap.Strict as HashMap
+import qualified Data.Map.Strict as Map
+import qualified Data.Sequence as Seq
+import qualified Data.Unique as Unique
+import qualified Network.HTTP.Client as Http
+
+import Radicle
+import qualified Radicle.Internal.MachineBackend.Ipfs as Ipfs
+import qualified Radicle.Internal.UUID as UUID
+import qualified Radicle.Ipfs as Ipfs
+
+jsonToValue :: Aeson.Value -> Aeson.Parser Value
+jsonToValue = Aeson.withText "Value" $ \t -> do
+ case parse "[daemon]" t of
+ Left err -> fail $ "failed to parse Radicle expression: " <> show err
+ Right v -> pure v
+
+valueToJson :: Value -> Aeson.Value
+valueToJson = Aeson.String . renderCompactPretty
+
+-- * Types
+
+newtype MachineId = MachineId { getMachineId :: Text }
+ deriving (Show, Eq, Ord, Generic)
+
+instance Aeson.ToJSONKey MachineId where
+ toJSONKey = Aeson.toJSONKeyText getMachineId
+instance Aeson.FromJSONKey MachineId where
+ fromJSONKey = Aeson.FromJSONKeyText MachineId
+instance Aeson.ToJSON MachineId where
+ toJSON (MachineId id) = Aeson.String id
+instance Aeson.FromJSON MachineId where
+ parseJSON js = MachineId <$> Aeson.parseJSON js
+
+data IpfsError
+ = IpfsDaemonError Ipfs.IpfsException
+ | InternalError Text
+ | NetworkError Text
+
+-- | Messages sent on a machine's IPFS pubsub topic.
+data Message = New InputsApplied | Submit SubmitInputs
+
+instance Aeson.FromJSON Message where
+ parseJSON = Aeson.withObject "Message" $ \o -> do
+ typ :: Text <- o .: "type"
+ case typ of
+ "new_inputs" -> do
+ nonce <- o .: "nonce"
+ results <- traverse jsonToValue =<< o .: "results"
+ pure $ New InputsApplied{..}
+ "request_inputs" -> do
+ nonce <- o .: "nonce"
+ expressions <- traverse jsonToValue =<< o .: "expressions"
+ pure $ Submit SubmitInputs{..}
+ _ -> fail "Messages on a machine's pubsub topic must be of type 'new_inputs' or 'request_inputs'."
+
+instance Aeson.ToJSON Message where
+ toJSON = \case
+ New InputsApplied{..} -> Aeson.object
+ [ "type" .= ("new_inputs" :: Text)
+ , "nonce" .= nonce
+ , "results" .= (valueToJson <$> results)
+ ]
+ Submit SubmitInputs{..} -> Aeson.object
+ [ "type" .= ("request_inputs" :: Text)
+ , "nonce" .= nonce
+ , "expressions" .= (valueToJson <$> expressions)
+ ]
+
+-- | Message sent to signal a new input has been added to the machine.
+data InputsApplied = InputsApplied
+ { nonce :: Maybe Text
+ , results :: [Value]
+ }
+
+-- | Message sent to request the writer add inputs to the machine.
+data SubmitInputs = SubmitInputs
+ { nonce :: Text
+ , expressions :: [Value]
+ }
+
+-- * Ipfs helpers
+
+-- TODO: All the error handling w.r.t interactions with IPFS should be
+-- done in "Radicle.Ipfs".
+safeIpfs :: forall a. IO a -> ExceptT IpfsError IO a
+safeIpfs io = ExceptT $ liftIO $
+ catches (Right <$> io)
+ [ Handler $ pure . Left . IpfsDaemonError
+ , Handler httpHdlr
+ ]
+ where
+ httpHdlr e = pure $ Left $ case e of
+ Http.HttpExceptionRequest _
+ (Http.StatusCodeException _
+ (decodeStrict ->
+ Just (Aeson.Object (HashMap.lookup "Message" ->
+ Just (Aeson.String msg))))) -> NetworkError msg
+ _ -> InternalError (toS (displayException e))
+
+
+-- | Write some inputs to an IPFS machine.
+-- TODO: we might want to make this safer by taking the expected head
+-- MachineEntryIndex as an argument.
+writeIpfs :: MachineId -> [Value] -> ExceptT IpfsError IO Ipfs.MachineEntryIndex
+writeIpfs (MachineId id) vs = safeIpfs $ Ipfs.sendIpfs id (Seq.fromList vs)
+
+newtype Topic = Topic { getTopic :: Text }
+
+-- | The IPFS pubsub topic associated to a machine.
+machineTopic :: MachineId -> Topic
+machineTopic (MachineId id) = Topic ("radicle:machine:" <> id)
+
+-- | Publish a 'Message' on a machine's IPFS pubsub topic.
+publish :: MachineId -> Message -> ExceptT IpfsError IO ()
+publish id msg = safeIpfs $
+ Ipfs.publish (getTopic (machineTopic id)) (Aeson.encode msg)
+
+-- | Subscribe to messages on a machine's IPFS pubsub topic.
+subscribeForever :: MachineId -> MsgHandler -> IO ()
+subscribeForever id messageHandler = Ipfs.subscribe topic pubsubHandler
+ where
+ Topic topic = machineTopic id
+ pubsubHandler Ipfs.PubsubMessage{..} = messageHandler $ case decodeStrict messageData of
+ Just msg -> Right msg
+ Nothing -> Left (toS messageData)
+
+-- | Get inputs of an IPFS machine from a certain index.
+machineInputsFrom :: MachineId -> Maybe Ipfs.MachineEntryIndex -> ExceptT IpfsError IO (Ipfs.MachineEntryIndex, [Value])
+machineInputsFrom (MachineId id) = safeIpfs . Ipfs.receiveIpfs id
+
+-- | Create an IPFS machine and return its ID.
+createMachine :: ExceptT Text IO MachineId
+createMachine = ExceptT $ liftIO $ second MachineId <$> (Ipfs.ipfsMachineCreate =<< UUID.uuid)
+
+-- * Topic subscriptions
+
+type MsgHandler = Either Text Message -> IO ()
+
+type RegisteredHandler = Unique.Unique
+
+newtype TopicSubscription = TopicSubscription
+ { handlers :: MVar (Map RegisteredHandler MsgHandler)
+ }
+
+-- | Given a machine ID, creates a subscription for that machine's
+-- pubsub topic. This allows adding and removing handlers for messages
+-- on that topic.
+initSubscription :: MachineId -> ExceptT IpfsError IO TopicSubscription
+initSubscription id = safeIpfs $ do
+ hdlrs <- newMVar Map.empty
+ _ <- async $ subscribeForever id (mainHdlr hdlrs)
+ pure $ TopicSubscription hdlrs
+ where
+ mainHdlr hdlrs msg = do
+ hs <- readMVar hdlrs
+ traverse_ ($ msg) (Map.elems hs)
+
+-- | Add a message handler to a topic subscription.
+addHandler :: TopicSubscription -> MsgHandler -> IO RegisteredHandler
+addHandler ts h = do
+ u <- Unique.newUnique
+ modifyMVar (handlers ts) (pure . (,()) . Map.insert u h)
+ pure u
+
+-- | Remove a message handler from a topic subscription.
+removeHandler :: TopicSubscription -> RegisteredHandler -> IO ()
+removeHandler ts u =
+ modifyMVar (handlers ts) (pure . (,()) . Map.delete u)
+
+-- | Block while waiting for a message which matches a
+-- predicate. Returns @Just msg@ where @msg@ is the first message
+-- passing the predicate if such a message arrives before the
+-- specified amount of milliseconds. Otherwise, returns @Nothing@.
+subscribeOne :: TopicSubscription -> Int64 -> (Message -> Bool) -> (Text -> IO ()) -> IO (Maybe Message)
+subscribeOne sub timeout pr badMsg = do
+ var <- newEmptyMVar
+ let onMsg = \case
+ Right msg | pr msg -> putMVar var (Just msg)
+ Right _ -> pure ()
+ Left bad -> badMsg bad
+ bracket
+ (addHandler sub onMsg)
+ (removeHandler sub)
+ (const $ forkIO (threadDelay (fromIntegral timeout * 1000) >> putMVar var Nothing) *> readMVar var)
diff --git a/src/Radicle/Internal/CLI.hs b/src/Radicle/Internal/CLI.hs
index 81387c0..e34e16e 100644
--- a/src/Radicle/Internal/CLI.hs
+++ b/src/Radicle/Internal/CLI.hs
@@ -1,5 +1,6 @@
module Radicle.Internal.CLI
- ( getHistoryFile
+ ( getRadicleFile
+ , getHistoryFile
)
where
@@ -8,15 +9,18 @@ import System.Directory
(XdgDirectory(..), createDirectoryIfMissing, getXdgDirectory)
import System.FilePath (takeDirectory)
--- | Location of the radicle history, usually
--- @~/.local/share/radicle/history@.
---
+-- | Location of a radicle related file.
-- Creates the parent directory if it does not exist.
--
-- See 'getXdgDirectory' 'XdgData' for how @~/.local/share@ is
-- determined.
-getHistoryFile :: IO FilePath
-getHistoryFile = do
- file <- getXdgDirectory XdgData "radicle/history"
+getRadicleFile :: FilePath -> IO FilePath
+getRadicleFile name = do
+ file <- getXdgDirectory XdgData $ "radicle/" <> name
createDirectoryIfMissing True (takeDirectory file)
pure file
+
+-- | Location of the radicle history, usually
+-- @~/.local/share/radicle/history@.
+getHistoryFile :: IO FilePath
+getHistoryFile = getRadicleFile "history"
diff --git a/src/Radicle/Internal/ConcurrentMap.hs b/src/Radicle/Internal/ConcurrentMap.hs
new file mode 100644
index 0000000..cb4a4e4
--- /dev/null
+++ b/src/Radicle/Internal/ConcurrentMap.hs
@@ -0,0 +1,83 @@
+module Radicle.Internal.ConcurrentMap
+ ( empty
+ , fromMap
+ , lookup
+ , nonAtomicRead
+ , insertNew
+ , modifyExistingValue
+ , modifyValue
+ , CMap
+ ) where
+
+import Protolude hiding (empty)
+
+import qualified Data.Map.Strict as Map
+
+-- | A Map which offers atomic operations on the values associated to
+-- keys. Assumes that the values are independent of one another; does
+-- not offer a consistent view over the values. Therefore this is best
+-- used when the keys uniquely identify the resources represented by
+-- the values.
+newtype CMap k v = CMap (MVar (Map k (MVar v)))
+
+-- | Create a new empty 'CMap'.
+empty :: IO (CMap k v)
+empty = CMap <$> newMVar Map.empty
+
+fromMap :: Map k v -> IO (CMap k v)
+fromMap m = CMap <$> (traverse newMVar m >>= newMVar)
+
+-- | Atomically lookup a value.
+lookup :: Ord k => k -> CMap k v -> IO (Maybe v)
+lookup k (CMap m_) = withMVar m_ $ \m -> do
+ case Map.lookup k m of
+ Nothing -> pure Nothing
+ Just v_ -> pure <$> readMVar v_
+
+-- | Non-atmoically read the contents of a 'CMap'. Provides a
+-- consistent shapshot of which keys were present at some
+-- time. However, the values might be snapshotted at different times.
+nonAtomicRead :: CMap k v -> IO (Map k v)
+nonAtomicRead (CMap m_) = do
+ m <- readMVar m_
+ traverse readMVar m
+
+-- | Atomically insert a key-value pair into a 'CMap' but only if the
+-- key is not already a member.
+insertNew :: Ord k => k -> v -> CMap k v -> IO (Maybe ())
+insertNew k v (CMap m_) = modifyMVar m_ $ \m ->
+ if Map.member k m
+ then pure (m, Nothing)
+ else do
+ v_ <- newMVar v
+ let m' = Map.insert k v_ m
+ pure (m', Just ())
+
+-- | Atomically modifies a value associated to a key but only if it
+-- exists. There is no guarantee the value is still associated with
+-- the same key, or any key, by the time the operation completes.
+modifyExistingValue :: Ord k => k -> CMap k v -> (v -> IO (v, a)) -> IO (Maybe a)
+modifyExistingValue k (CMap m_) f = do
+ m <- readMVar m_
+ case Map.lookup k m of
+ Nothing -> pure Nothing
+ Just v_ -> do
+ x <- modifyMVar v_ f
+ pure (Just x)
+
+-- | Atomically modifies the value associated to a key.
+modifyValue :: Ord k => k -> CMap k v -> (Maybe v -> IO (Maybe v, a)) -> IO a
+modifyValue k (CMap m_) f = modifyMVar m_ $ \m ->
+ case Map.lookup k m of
+ Nothing -> do
+ (maybev, x) <- f Nothing
+ case maybev of
+ Just v -> do
+ v_ <- newMVar v
+ pure (Map.insert k v_ m, x)
+ Nothing -> pure (m, x)
+ Just v_ -> modifyMVar v_ $ \v -> do
+ (maybev', x) <- f (Just v)
+ case maybev' of
+ Just v' -> pure (v', (m, x))
+ Nothing -> pure (v, (Map.delete k m, x))
diff --git a/src/Radicle/Internal/Core.hs b/src/Radicle/Internal/Core.hs
index 2b11b87..55cc06f 100644
--- a/src/Radicle/Internal/Core.hs
+++ b/src/Radicle/Internal/Core.hs
@@ -11,7 +11,7 @@ import Codec.Serialise (Serialise)
import Control.Monad.Except
(ExceptT(..), MonadError, runExceptT, throwError)
import Control.Monad.State hiding (State)
-import Data.Aeson (FromJSON(..), ToJSON(..))
+import Data.Aeson (ToJSON(..))
import qualified Data.Aeson as A
import Data.Copointed (Copointed(..))
import qualified Data.HashMap.Strict as HashMap
@@ -402,18 +402,6 @@ isAtom :: Value -> Maybe Ident
isAtom (Atom i) = pure i
isAtom _ = Nothing
-instance A.FromJSON Value where
- parseJSON = \case
- A.Number n -> pure $ Number (toRational n)
- A.String s -> pure $ String s
- A.Array ls -> List . toList <$> traverse parseJSON ls
- A.Bool b -> pure $ Boolean b
- A.Null -> pure $ Keyword (Ident "null")
- A.Object hm -> do
- let kvs = HashMap.toList hm
- vs <- traverse parseJSON (snd <$> kvs)
- pure . Dict . Map.fromList $ zip (String . fst <$> kvs) vs
-
-- | Convert a radicle `Value` into an 'aeson' value, if possible.
--
-- >>> import Data.Aeson (encode)
diff --git a/src/Radicle/Internal/MachineBackend/Ipfs.hs b/src/Radicle/Internal/MachineBackend/Ipfs.hs
index d6a3414..a301841 100644
--- a/src/Radicle/Internal/MachineBackend/Ipfs.hs
+++ b/src/Radicle/Internal/MachineBackend/Ipfs.hs
@@ -15,10 +15,7 @@
-- }
-- @
--
-module Radicle.Internal.MachineBackend.Ipfs
- ( ipfsPrimFns
- , ipfsMachineCreate
- ) where
+module Radicle.Internal.MachineBackend.Ipfs where
import Protolude hiding (TypeError, catch, try)
diff --git a/src/Radicle/Ipfs.hs b/src/Radicle/Ipfs.hs
index 0402037..de1a5fe 100644
--- a/src/Radicle/Ipfs.hs
+++ b/src/Radicle/Ipfs.hs
@@ -13,26 +13,38 @@ module Radicle.Ipfs
, DagPutResponse(..)
, dagPut
-
, dagGet
, namePublish
, NameResolveResponse(..)
, nameResolve
+
+ , PubsubMessage(..)
+ , publish
+ , subscribe
) where
import Protolude hiding (TypeError, catch, try)
import Control.Exception.Safe
import Control.Monad.Fail
+import Control.Monad.Trans.Resource
import Data.Aeson (FromJSON, ToJSON, (.:), (.=))
import qualified Data.Aeson as Aeson
import qualified Data.Aeson.Types as Aeson
+import qualified Data.ByteString.Multibase as Multibase
+import Data.Conduit ((.|))
+import qualified Data.Conduit as C
+import qualified Data.Conduit.Attoparsec as C
+import qualified Data.Conduit.Combinators as C
import Data.IPLD.CID
import qualified Data.Text as T
+import qualified Data.Text.Encoding as T
import Lens.Micro ((.~), (^.))
import Network.HTTP.Client
(HttpException(..), HttpExceptionContent(..))
+import qualified Network.HTTP.Client as HTTP
+import qualified Network.HTTP.Conduit as HTTP
import qualified Network.Wreq as Wreq
import System.Environment (lookupEnv)
@@ -103,6 +115,55 @@ addressFromText t =
-- * IPFS node API
--------------------------------------------------------------------------
+data PubsubMessage = PubsubMessage
+ { messageTopicIDs :: [Text]
+ , messageData :: ByteString
+ , messageFrom :: ByteString
+ , messageSeqno :: ByteString
+ } deriving (Eq, Show)
+
+instance FromJSON PubsubMessage where
+ parseJSON = Aeson.withObject "PubsubMessage" $ \o -> do
+ messageTopicIDs <- o .: "topicIDs"
+ Right messageData <- o .: "data" <&> T.encodeUtf8 <&> Multibase.decodeBase64
+ Right messageFrom <- o .: "from" <&> T.encodeUtf8 <&> Multibase.decodeBase64
+ Right messageSeqno <- o .: "seqno" <&> T.encodeUtf8 <&> Multibase.decodeBase64
+ pure PubsubMessage {..}
+
+-- | Subscribe to a topic and call @messageHandler@ on every message.
+-- The IO action blocks while we are subscribed. To stop subscription
+-- you need to kill the thread the subscription is running in.
+subscribe :: Text -> (PubsubMessage -> IO ()) -> IO ()
+subscribe topic messageHandler = runResourceT $ do
+ mgr <- liftIO $ HTTP.newManager HTTP.defaultManagerSettings
+ req <- HTTP.parseRequest "http://localhost:9301/api/v0/pubsub/sub" <&>
+ HTTP.setQueryString
+ [ ("arg", Just $ T.encodeUtf8 topic)
+ , ("encoding", Just "json")
+ , ("stream-channels", Just "true")
+ ]
+ body <- HTTP.responseBody <$> HTTP.http req mgr
+ C.runConduit $ body .| fromJSONC .| C.mapM_ (liftIO . messageHandler) .| C.sinkNull
+ pure ()
+ where
+ fromJSONC :: (MonadThrow m, Aeson.FromJSON a) => C.ConduitT ByteString a m ()
+ fromJSONC = jsonC .| C.mapM parseThrow
+
+ jsonC :: (MonadThrow m) => C.ConduitT ByteString Aeson.Value m ()
+ jsonC = C.peekForever (C.sinkParser Aeson.json >>= C.yield)
+
+ parseThrow :: (MonadThrow m, Aeson.FromJSON a) => Aeson.Value -> m a
+ parseThrow value = do
+ case Aeson.fromJSON value of
+ Aeson.Error err -> throwString err
+ Aeson.Success a -> pure a
+
+
+-- | Publish a message to a topic.
+publish :: Text -> LByteString -> IO ()
+publish topic message =
+ void $ ipfsHttpPost' "pubsub/pub" [("arg", topic)] "data" message
+
newtype KeyGenResponse = KeyGenResponse IpnsId
keyGen :: Text -> IO KeyGenResponse
@@ -180,12 +241,21 @@ ipfsHttpPost
-> LByteString -- ^ Payload argument
-> IO a
ipfsHttpPost path params payloadArgName payload = do
- let opts = Wreq.defaults & Wreq.params .~ params
- url <- ipfsApiUrl path
- res <- Wreq.postWith opts (toS url) (Wreq.partLBS payloadArgName payload) `catch` handleRequestException
+ res <- ipfsHttpPost' path params payloadArgName payload
jsonRes <- Wreq.asJSON res `catch` handleParseException path
pure $ jsonRes ^. Wreq.responseBody
+ipfsHttpPost'
+ :: Text -- ^ Path of the endpoint under "/api/v0/"
+ -> [(Text, Text)] -- ^ URL query parameters
+ -> Text -- ^ Name of the argument for payload
+ -> LByteString -- ^ Payload argument
+ -> IO (Wreq.Response LByteString)
+ipfsHttpPost' path params payloadArgName payload = do
+ let opts = Wreq.defaults & Wreq.params .~ params
+ url <- ipfsApiUrl path
+ Wreq.postWith opts (toS url) (Wreq.partLBS payloadArgName payload) `catch` handleRequestException
+
ipfsApiUrl :: Text -> IO Text
ipfsApiUrl path = do
baseUrl <- fromMaybe "http://localhost:9301" <$> lookupEnv "RAD_IPFS_API_URL"