The document discusses using Free monads and STM (Software Transactional Memory) for building concurrent applications in Haskell. It provides examples of applications like a labyrinth game and meteor shower simulation that are built using Free monads, STM transactions, processes and signals to coordinate concurrent operations. The document also discusses building a blockchain application using a similar approach with nodes, services, state and declarative protocols.

1. Concurrent applications
with Free monads and STM
Haskell Love 2020
1
Alexander Granin
graninas@gmail.com
Twitter: @graninas

2. Plan
● In love with Free monads and STM
● Testing?
● How does this magic work?
● Performance?
● Conclusion: The truths I learned
● (Bonus) No, really, how does this magic work?
2

3. In Love with Free monads and STM
3

4. 4
Labyrinth game (Hydra framework)

5. 5
printLabyrinth :: AppState -> LangL ()
printLabyrinth state = do
lab <- readVarIO (state ^. labyrinth)
plPos <- readVarIO (state ^. playerPos)
let labRender = renderLabyrinth lab plPos
printLabyrinthRender labRender
data AppState = AppState
{ _labyrinth :: StateVar Labyrinth
, _playerPos :: StateVar PlayerPos
, _gameState :: StateVar GameState
}
Labyrinth game

6. 6
printLabyrinth :: AppState -> LangL ()
printLabyrinth state = do
lab <- readVarIO (state ^. labyrinth)
plPos <- readVarIO (state ^. playerPos)
let labRender = renderLabyrinth lab plPos
printLabyrinthRender labRender
data AppState = AppState
{ _labyrinth :: StateVar Labyrinth
, _playerPos :: StateVar PlayerPos
, _gameState :: StateVar GameState
}
Labyrinth game
Free monadic
scenario

7. 7
printLabyrinth :: AppState -> LangL ()
printLabyrinth state = do
lab <- readVarIO (state ^. labyrinth)
plPos <- readVarIO (state ^. playerPos)
let labRender = renderLabyrinth lab plPos
printLabyrinthRender labRender
data AppState = AppState
{ _labyrinth :: StateVar Labyrinth
, _playerPos :: StateVar PlayerPos
, _gameState :: StateVar GameState
}
-- | Concurrent variable
newtype StateVar a = StateVar
{ _varId :: Int
}
Labyrinth game

8. 8
printLabyrinth :: AppState -> LangL ()
printLabyrinth state = do
lab <- readVarIO (state ^. labyrinth)
plPos <- readVarIO (state ^. playerPos)
let labRender = renderLabyrinth lab plPos
printLabyrinthRender labRender
data AppState = AppState
{ _labyrinth :: StateVar Labyrinth
, _playerPos :: StateVar PlayerPos
, _gameState :: StateVar GameState
}
-- | Concurrent variable
newtype StateVar a = StateVar
{ _varId :: Int
}
Labyrinth game
Magic!
(Typed Avatar pattern)

9. 9
Labyrinth game
labyrinthApp :: AppState -> AppL ()
labyrinthApp st = do
scenario $ putStrLn "Labyrinth (aka Terra Incognita) game"
cliToken <- cli (onStep st) (onUnknownCommand st) $ do
cmd "go up" $ onPlayerMove st $ makePlayerMove st DirUp
cmd "go down" $ onPlayerMove st $ makePlayerMove st DirDown
cmd "go left" $ onPlayerMove st $ makePlayerMove st DirLeft
cmd "go right" $ onPlayerMove st $ makePlayerMove st DirRight
cmd "quit" $ quit st

10. 10
Labyrinth game
labyrinthApp :: AppState -> AppL ()
labyrinthApp st = do
scenario $ putStrLn "Labyrinth (aka Terra Incognita) game"
cliToken <- cli (onStep st) (onUnknownCommand st) $ do
cmd "go up" $ onPlayerMove st $ makePlayerMove st DirUp
cmd "go down" $ onPlayerMove st $ makePlayerMove st DirDown
cmd "go left" $ onPlayerMove st $ makePlayerMove st DirLeft
cmd "go right" $ onPlayerMove st $ makePlayerMove st DirRight
cmd "quit" $ quit st
Control Structure pattern

11. 11
Labyrinth game
labyrinthApp :: AppState -> AppL ()
labyrinthApp st = do
scenario $ putStrLn "Labyrinth (aka Terra Incognita) game"
cliToken <- cli (onStep st) (onUnknownCommand st) $ do
cmd "go up" $ onPlayerMove st $ makePlayerMove st DirUp
cmd "go down" $ onPlayerMove st $ makePlayerMove st DirDown
cmd "go left" $ onPlayerMove st $ makePlayerMove st DirLeft
cmd "go right" $ onPlayerMove st $ makePlayerMove st DirRight
cmd "quit" $ quit st
atomically $ do
finished <- readVar $ cliFinishedToken cliToken
when (not finished) retry
Free monadic eDSL over STM
Control Structure pattern

12. 12
MeteorCounter app (Hydra framework)

13. 13
MeteorCounter app (Hydra framework)
meteorsMonitoring :: AppConfig -> AppL ()
meteorsMonitoring cfg = do
st <- atomically $ initState cfg
process $ forever $ meteorCounter st
process $ forever $ withRandomDelay st $ meteorShower st NorthEast
process $ forever $ withRandomDelay st $ meteorShower st NorthWest
process $ forever $ withRandomDelay st $ meteorShower st SouthEast
process $ forever $ withRandomDelay st $ meteorShower st SouthWest
atomically $ do
let totalVar :: StateVar Int = _totalMeteors st
total <- readVar totalVar
when (total < 1000) retry

14. 14
MeteorCounter app (Hydra framework)
meteorsMonitoring :: AppConfig -> AppL ()
meteorsMonitoring cfg = do
st <- atomically $ initState cfg
process $ forever $ meteorCounter st
process $ forever $ withRandomDelay st $ meteorShower st NorthEast
process $ forever $ withRandomDelay st $ meteorShower st NorthWest
process $ forever $ withRandomDelay st $ meteorShower st SouthEast
process $ forever $ withRandomDelay st $ meteorShower st SouthWest
atomically $ do
let totalVar :: StateVar Int = _totalMeteors st
total <- readVar totalVar
when (total < 1000) retry
Processes

15. 15
MeteorCounter app (Hydra framework)
meteorsMonitoring :: AppConfig -> AppL ()
meteorsMonitoring cfg = do
st <- atomically $ initState cfg
process $ forever $ meteorCounter st
process $ forever $ withRandomDelay st $ meteorShower st NorthEast
process $ forever $ withRandomDelay st $ meteorShower st NorthWest
process $ forever $ withRandomDelay st $ meteorShower st SouthEast
process $ forever $ withRandomDelay st $ meteorShower st SouthWest
atomically $ do
let totalVar :: StateVar Int = _totalMeteors st
total <- readVar totalVar
when (total < 1000) retry
Finish condition

16. 16
Blockchain app (Node framework)
Graph Node
Graph Node
PoW Node
TCP
UDP
JSON RPC

17. 17
Blockchain app
graphNode :: NodeConfig GraphNode -> GraphNodeData -> NodeDefinitionL ()
graphNode cfg nodeData = do
let gsData = nodeData ^. graphServiceData
serving Udp (cfg ^. nodeUdpPort) $ do
handler $ methodPing
handler $ acceptKBlock gsData
process $ forever $ do
awaitSignal $ gsData ^. dumpToDBSignal
dumpToDB gsData
process $ forever $ do
awaitSignal $ gsData ^. restoreFromDBSignal
restoreFromDB gsData
std $ do
stdHandler $ stopNodeHandler nodeData
awaitNodeFinished nodeData

18. 18
Blockchain app
graphNode :: NodeConfig GraphNode -> GraphNodeData -> NodeDefinitionL ()
graphNode cfg nodeData = do
let gsData = nodeData ^. graphServiceData
serving Udp (cfg ^. nodeUdpPort) $ do
handler $ methodPing
handler $ acceptKBlock gsData
process $ forever $ do
awaitSignal $ gsData ^. dumpToDBSignal
dumpToDB gsData
process $ forever $ do
awaitSignal $ gsData ^. restoreFromDBSignal
restoreFromDB gsData
std $ do
stdHandler $ stopNodeHandler nodeData
awaitNodeFinished nodeData
UDP Service
(Declarative
Protocol pattern)

19. 19
Blockchain app
graphNode :: NodeConfig GraphNode -> GraphNodeData -> NodeDefinitionL ()
graphNode cfg nodeData = do
let gsData = nodeData ^. graphServiceData
serving Udp (cfg ^. nodeUdpPort) $ do
handler $ methodPing
handler $ acceptKBlock gsData
process $ forever $ do
awaitSignal $ gsData ^. dumpToDBSignal
dumpToDB gsData
process $ forever $ do
awaitSignal $ gsData ^. restoreFromDBSignal
restoreFromDB gsData
std $ do
stdHandler $ stopNodeHandler nodeData
awaitNodeFinished nodeData
Background
dump / restore
processes

20. 20
Blockchain app
graphNode :: NodeConfig GraphNode -> GraphNodeData -> NodeDefinitionL ()
graphNode cfg nodeData = do
let gsData = nodeData ^. graphServiceData
serving Udp (cfg ^. nodeUdpPort) $ do
handler $ methodPing
handler $ acceptKBlock gsData
process $ forever $ do
awaitSignal $ gsData ^. dumpToDBSignal
dumpToDB gsData
process $ forever $ do
awaitSignal $ gsData ^. restoreFromDBSignal
restoreFromDB gsData
std $ do
stdHandler $ stopNodeHandler nodeData
awaitNodeFinished nodeData
CLI
& Node finish condition

21. 21
Blockchain app state
data GraphNodeData = GraphNodeData
{ _graphServiceData :: GraphServiceData
, _status :: StateVar NodeStatus
}
data GraphServiceData = GraphServiceData
{ _blockchain :: BlockchainData
, _db :: Maybe DBModel
, _dumpToDBSignal :: StateVar Bool
, _restoreFromDBSignal :: StateVar Bool
, _checkPendingSignal :: StateVar Bool
}
Node
State
Node
Configs
Scenarios (Free monad chains)
Process Process
Graph
Process
STM transactions

22. 22
Blockchain app state
data GraphNodeData = GraphNodeData
{ _graphServiceData :: GraphServiceData
, _status :: StateVar NodeStatus
}
data GraphServiceData = GraphServiceData
{ _blockchain :: BlockchainData
, _db :: Maybe DBModel
, _dumpToDBSignal :: StateVar Bool
, _restoreFromDBSignal :: StateVar Bool
, _checkPendingSignal :: StateVar Bool
}
awaitSignal :: (Monad m, StateIO m) => StateVar Bool -> m ()
awaitSignal signalVar = do
atomically $ unlessM (readVar signalVar) retry
writeVarIO signalVar False
Signal vars

23. 23
Concurrent STM-based graph
Active window
Request
Request
Request
KV DB
STM-based transactions

24. 24
addMBlock :: WindowedGraph -> Microblock -> StateL Bool
addMBlock wndGraph mblock@(Microblock hash _ _ _) = do
kblock <- getKBlock wndGraph hash
unless (isJust kblock) $ logInfo $ "Can't add MBlock: KBlock not found (" +|| hash ||+ ")."
when (isJust kblock) $ do
logInfo $ "Adding MBlock to the graph for KBlock (" +|| hash ||+ ")."
evalGraph wndGraph $ do
newNode (MBlockContent mblock)
newLink hash (MBlockContent mblock)
pure $ isJust kblock
Graph code sample

25. 25
addMBlock :: WindowedGraph -> Microblock -> StateL Bool
addMBlock wndGraph mblock@(Microblock hash _ _ _) = do
kblock <- getKBlock wndGraph hash
unless (isJust kblock) $ logInfo $ "Can't add MBlock: KBlock not found (" +|| hash ||+ ")."
when (isJust kblock) $ do
logInfo $ "Adding MBlock to the graph for KBlock (" +|| hash ||+ ")."
evalGraph wndGraph $ do
newNode (MBlockContent mblock)
newLink hash (MBlockContent mblock)
pure $ isJust kblock
Graph code sample
STM-like transaction eDSL
newVar
readVar
writeVar
retry
+Logger interface

26. 26
addMBlock :: WindowedGraph -> Microblock -> StateL Bool
addMBlock wndGraph mblock@(Microblock hash _ _ _) = do
kblock <- getKBlock wndGraph hash
unless (isJust kblock) $ logInfo $ "Can't add MBlock: KBlock not found (" +|| hash ||+ ")."
when (isJust kblock) $ do
logInfo $ "Adding MBlock to the graph for KBlock (" +|| hash ||+ ")."
evalGraph wndGraph $ do
newNode (MBlockContent mblock)
newLink hash (MBlockContent mblock)
pure $ isJust kblock
Graph code sample
Work with graph
NewNode
DeleteNode
NewLink
DeleteLink
GetNode
ClearGraph

27. 27
addMBlock :: WindowedGraph -> Microblock -> StateL Bool
addMBlock wndGraph mblock@(Microblock hash _ _ _) = do
kblock <- getKBlock wndGraph hash
unless (isJust kblock) $ logInfo $ "Can't add MBlock: KBlock not found (" +|| hash ||+ ")."
when (isJust kblock) $ do
logInfo $ "Adding MBlock to the graph for KBlock (" +|| hash ||+ ")."
evalGraph wndGraph $ do
newNode (MBlockContent mblock)
newLink hash (MBlockContent mblock)
pure $ isJust kblock
Graph code sample
Logging within the transaction

28. Testing?
28
♡

29. 29
spec :: Spec
spec = unstableTest $ slowTest
$ describe "PoW and graph node interaction"
$ fromHUnitTest $ TestList
[ TestLabel "Accept kblocks produced in order" $ testAcceptKblock InOrder
, TestLabel "Accept kblocks produced in random order" $ testAcceptKblock RandomOrder
]
testAcceptKblock :: Ordering -> Test
testAcceptKblock order = TestCase $ withNodesManager $ mgr -> do
void $ startNode mgr $ graphNode graphNodeTransmitterConfig
waitForNode transmitterRpcAddress
void $ startNode mgr $ powNode $ genPoWNodeConfig { _kblocksOrder = order}
waitForNode powRpcAddress
void $ makeIORpcRequest powRpcAddress $ NBlockPacketGeneration kblockCount timeGap
let predicate kBlock = kBlock ^. number == kblockCount
void $ makeRpcRequestWithPredicate predicate transmitterRpcAddress GetLastKBlock
Testing nodes (virtual environment)

30. 30
spec :: Spec
spec = do
around (withCoreRuntime Nothing) // Arrange
$ it "generated labyrinth has correct bounds"
$ runtime -> property
$ withMaxSuccess 5
$ monadicIO
$ do
eLab <- run $ try $ runLangL runtime generateRndLabyrinth // Act
case eLab of
Left (err :: SomeException) -> assert False // Assert
Right lab -> do
let LabyrinthInfo {liBounds, liWormholes} = analyzeLabyrinth lab
let (x, y) = liBounds
let wms = Map.size liWormholes
assert $ x * y >= 16 && x * y <= 100
assert $ (wms >= 2) && (wms <= 5)
Testing the labyrinth logic (black box, property-based)

31. 31
spec :: Spec
spec = do
around (withAppRuntime Nothing) $ // Arrange
describe "Labyrinth generation tests" $
it "generateLabyrinth" $ rt -> do
lab <- runLangL (_coreRuntime rt) $ generateLabyrinth (4, 4) 3 5 // Act
let LabyrinthInfo {liBounds, liWormholes, liExits} = analyzeLabyrinth lab
liBounds `shouldBe` (4, 4) // Assert
(Map.size liWormholes) `shouldSatisfy` (x -> x >= 2 && x <= 5)
(Set.size liExits) `shouldSatisfy` (x -> x >= 1 && x <= 3)
Testing the labyrinth logic (black box, integration)

32. How does this magic work?
32

33. It works nicely.
33

34. It works nicely.
34
Business logic code is very simple.

35. It works nicely.
35
Business logic code is very simple.
Testing is very simple.

36. It works nicely.
36
Business logic code is very simple.
Testing is very simple.
Complete separation of concerns.

37. Performance?
37

38. 38
Performance of Free monads (dumb testing approach)
Ops cnt FT FreeM ChurchM IO
10 0.265 0.222 0.223 0.227
100 0.221 0.226 0.228 0.222
1000 0.227 0.245 0.223 0.226
10000 0.229 4.106 0.227 0.224
100000 0.289 inf 0.31 0.309
1000000 0.859 inf 1.134 0.857
10000000 6.384 inf 9.507 6.413
20000000 13.734 inf 18.997 12.588
30000000 18.16 inf 28.568 17.76
flow :: IORef Int -> AppL ()
flow ref = scenario $ do
val' <- evalIO $ readIORef ref
val <- getRandomInt (1, 100)
evalIO $ writeIORef ref $ val' + val
scenario :: Int -> AppRuntime -> IO ()
scenario ops appRt = do
ref <- newIORef 0
void $ startApp appRt (replicateM_ ops $ flow ref)
val <- readIORef ref
print val
$ time PerfTestApp2
https://github.com/graninas/Hydra/tree/master/app/PerfTestApp2

39. The truths I learned

40. PerformanceGood Design >

41. Performance
Perfection
Good Design
Getting Things Done
>
>

42. Performance
Perfection
Correctness
Good Design
Getting Things Done
Tests
>
>
>

43. Performance
Perfection
Correctness
Math Beauty
Good Design
Getting Things Done
Tests
Simplicity
>
>
>
>

44. Performance
Perfection
Correctness
Math Beauty
Technologies
Good Design
Getting Things Done
Tests
Simplicity
People
>
>
>
>
>

45. Performance
Perfection
Correctness
Math Beauty
Technologies
Belief
Good Design
Getting Things Done
Tests
Simplicity
People
Experiment
>
>
>
>
>
>

46. 46
GitHub List: Software Design in Haskell
https://github.com/graninas/software-design-in-haskell👉
Hydra: framework for backend & console apps
https://github.com/graninas/Hydra👉
Node: framework for distributed apps & blockchains
https://github.com/graninas/Node👉

47. graninas@gmail.com
graninas
graninas_channel
StrangeMooder
granin.alexander
Alexander Granin
47
Free monads STM
http://leanpub.com/functional-design-and-archit
ecture/c/4vpQNPcmRfo1
66% discount ($60 $20)
(expires 2020-08-05)

48. 48
Additional Materials
● Automatic White-Box Testing with Free Monads | Article, Showcase
● Hierarchical Free Monads and Software Design in Haskell | Talk
● Hierarchical Free Monads: The Most Developed Approach In Haskell
● Functional Design and Architecture | Book
● Hydra | Framework (web services with SQL/KV DB and concurrency)
● Node | Framework (distributed apps with KV DB and concurrency)
● Final Tagless vs Free Monads | Talk
● Software Design in Haskell | List of materials

49. No, really, how does this magic work?
49

50. 50
Hydra Framework
Implementation & Runtime
Interfaces
(Hierarchical Free Monad eDSLs)
Domain & Business Logic
ProcessL
Runtime, Interpreters, Configs, Environment
Application
Domain
Model
Business
Logic
DB Model
AppL
SqlDBL
LangL
Beam
LoggerL
StateL

51. 51
newtype StateVar a = StateVar { _varId :: Int }
data StateF next where
NewVar :: a -> (StateVar a -> next) -> StateF next
ReadVar :: StateVar a -> (a -> next) -> StateF next
WriteVar :: StateVar a -> a -> (() -> next) -> StateF next
Retry :: (a -> next) -> StateF next
EvalGraph :: TGraph c -> Free (HGraphF (TNodeL c)) x -> (x -> next) -> StateF next
EvalDelayedLogger :: LoggerL () -> (() -> next) -> StateF next
type StateL = Free StateF
STM-like Free interface

52. 52
STM-like Free language -> STM
newVar' :: StateRuntime -> a -> STM VarId
newVar' stateRt !a = do
nodeState <- takeTMVar $ stateRt ^. state
varId <- getVarId stateRt
tvar <- newTVar $ unsafeCoerce a
let !newMap = Map.insert varId (VarHandle tvar) nodeState
putTMVar (stateRt ^. state) newMap
pure varId
interpretStateF :: StateRuntime -> StateF a -> STM a
interpretStateF _ (Retry _) = retry
interpretStateF stateRt (NewVar !val next) = do
var <- newVar' stateRt val
pure $ next $ StateVar var
runStateL :: StateRuntime -> StateL a -> STM a
runStateL stateRt = foldFree (interpretStateF stateRt)

53. 53
Container language
data LangF next where
EvalLogger :: LoggerL () -> (() -> next) -> LangF next
EvalIO :: IO a -> (a -> next) -> LangF next
...
EvalStateAtomically :: StateL a -> (a -> next) -> LangF next
type LangL = Free LangF
atomically :: StateL a -> LangL a
atomically action = liftF $ EvalStateAtomically action id

54. 54
Upper level language -> IO
interpretLangF :: CoreRuntime -> LangF a -> IO a
interpretLangF coreRt (EvalStateAtomically action next) = do
let stateRt = coreRt ^. stateRuntime
let logHndl = coreRt ^. loggerRuntime . hsLoggerHandle
res <- atomically $ runStateL stateRt action
flushStmLogger (stateRt ^. stmLog) logHndl
pure $ next res
interpretLangF _ (EvalIO f next) = do
!r <- f
pure $ next r
interpretLangF coreRt (EvalLogger loggerAct next) =
next <$> runLoggerL (coreRt ^. loggerRuntime . hsLoggerHandle) loggerAct

55. 55
Behind the scenes...
newtype VarHandle = VarHandle (TVar Any)
data StateRuntime = StateRuntime
{ _varId :: TVar VarId -- ^ Var id counter
, _state :: TMVar (Map VarId VarHandle) -- ^ Tracked variables
, _stmLog :: TVar Log -- ^ Stm log entries
}
data CoreRuntime = CoreRuntime
{ _stateRuntime :: StateRuntime
, _rocksDBs :: RocksDBHandles
, _redisConns :: RedisConnections
, _loggerRuntime :: LoggerRuntime
, _processRuntime :: ProcessRuntime
, _sqlConns :: MVar (Map ConnTag NativeSqlConn)
, _httpClientManager :: Manager
, _cmdVerbosity :: CmdVerbosity
}