Tame cloud complexity with F#-powered DSLs

Yan Cui
Yan CuiSpeaker at Self
@theburningmonk
tame
|> (cloud <| complexity)
|> with
|> (fsharp >> powered >> DSLs)
Principal Engineer
Tame cloud complexity with F#-powered DSLs
Tame cloud complexity with F#-powered DSLs
image by nerovivo license : https://creativecommons.org/licenses/by-sa/2.0/
@theburningmonk
Under-Abstraction
@theburningmonk
Oversimplification
@theburningmonk
Impedance Mismatch
@theburningmonk
Amazon
DynamoDB
Amazon
SimpleWorkflow
Amazon
CloudWatch
CASE STUDY
@theburningmonk
F# DSLs. Awesome!
@theburningmonk
Amazon
DynamoDB
Amazon
SimpleW
Amazon
CloudWatch
CASE STUDY
@theburningmonk
@theburningmonk
managed
key-value store
@theburningmonk
redundancy
9-9s guarantee
@theburningmonk
great performance
@theburningmonk
name your
throughput
@theburningmonk
@theburningmonk
can be changed on-the-fly
@theburningmonk
@theburningmonk
infinitely scalable
(but you still have to pay for it)
@theburningmonk
Hash Key
Range Key
@theburningmonk
Query:
given a hash key
filter on
range key, or
local secondary index
@theburningmonk
Hash Key Range Key
Local Secondary Index
Global Secondary Index
@theburningmonk
Scan:
FULL TABLE search
(performance + cost concern)
@theburningmonk
Hash Key Range Key
Local Secondary Index
Global Secondary Index
Hash Key Range Key
Local Secondary Index
Who are the TOP 3 players in “Starship
X” with a score of at least 1000?
Global Secondary Index
@theburningmonk
@theburningmonk
select GameTitle, UserId, TopScore
from GameScores
where GameTitle = “Starship X”
and TopScore >= 1000
order desc
limit 3
with (NoConsistentRead, Index(GameTitleIndex, true))
DynamoDB.SQL
github.com/fsprojects/DynamoDb.SQL
@theburningmonk
GOAL
Disguise
complexity
@theburningmonk
GOAL
Prevent
abstraction leak
@theburningmonk
GOAL
SELECT UserId, TopScore
FROM GameScore
WHERE GameTitle CONTAINS “Zelda”
ORDER DESC
LIMIT 3
WITH (NoConsistentRead)
@theburningmonk
Query
AST
Execution
F# & FParsec*
*www.quanttec.com/fparsec
External DSL
via
@theburningmonk
@theburningmonk
SELECT * FROM GameScore
Abstract Syntax Tree (AST)
FParsec
@theburningmonk
SELECT * FROM GameScore
keyword keyword
* | attribute, attribute, …
table name
@theburningmonk
SELECT * FROM GameScore
type Attributes =
| Asterisk
| Attributes of string[]
@theburningmonk
SELECT * FROM GameScore
type Query =
{
Attributes : Attributes
Table : string
}
@theburningmonk
SELECT * FROM GameScore
Parser for “SELECT” keyword
pSelect
@theburningmonk
SELECT * FROM GameScore
pSelect
let pSelect = skipStringCI "select"
@theburningmonk
SELECT * FROM GameScore
pSelect
let pSelect = skipStringCI "select"
matches the string “select” (Case
Insensitive) and ignores it
@theburningmonk
SELECT * FROM GameScore
pFrom
let pFrom = skipStringCI "from"
@theburningmonk
SELECT * FROM GameScore
Parser for a string that
represents the table name
pTableName
@theburningmonk
SELECT * FROM GameScore
pTableName
let isTableName = isLetter <||> isDigit
let pTableName =
many1Satisfy isTableName
@theburningmonk
SELECT * FROM GameScore
pTableName
let isTableName = isLetter <||> isDigit
let pTableName =
many1Satisfy isTableName
@theburningmonk
SELECT * FROM GameScore
pTableName
let isTableName = isLetter <||> isDigit
let pTableName =
many1Satisfy isTableName
@theburningmonk
SELECT * FROM GameScore
pTableName
let isTableName = isLetter <||> isDigit
let pTableName =
many1Satisfy isTableName
@theburningmonk
SELECT * FROM GameScore
pTableName
let isTableName = isLetter <||> isDigit
let pTableName =
many1Satisfy isTableName
parses a sequence of one or
more chars that satisfies the
predicate function
@theburningmonk
SELECT * FROM GameScore
pAsterisk
*
pAttributeName
UserId, GameTitle, TopScore, …
@theburningmonk
SELECT * FROM GameScore
pAsterisk
*
pAttributeName
UserId, GameTitle, TopScore, …
let pAsterisk = stringCIReturn "*" Asterisk
@theburningmonk
SELECT * FROM GameScore
pAsterisk
*
pAttributeName
UserId, GameTitle, TopScore, …
let pAsterisk = stringCIReturn "*" Asterisk
matches the specified string
and return the given value
@theburningmonk
SELECT * FROM GameScore
pAsterisk
*
pAttributeName
UserId, GameTitle, TopScore, …
let isAttributeName = isLetter <||> isDigit
let pAttributeName =
many1Satisfy isAttributeName
@theburningmonk
SELECT * FROM GameScore
UserId, GameTitle, TopScore, …
pAttributeName pCommapAsterisk
*
let pComma = skipStringCI ","
@theburningmonk
SELECT * FROM GameScore
UserId, GameTitle, TopScore, …
pAttributeName pCommapAsterisk
*
let pAttributeNames =
sepBy1 pAttributeName pComma
@theburningmonk
SELECT * FROM GameScore
UserId, GameTitle, TopScore, …
pAttributeName pCommapAsterisk
*
let pAttributeNames =
sepBy1 pAttributeName pComma
parses one or more occurrences of
pAttributeName separated by pComma
@theburningmonk
SELECT * FROM GameScore
UserId, GameTitle, TopScore, …
pAttributeName pComma
sepBy1
pAttributeNames
pAsterisk
*
@theburningmonk
SELECT * FROM GameScore
pAsterisk
*
pAttributeNames
UserId, GameTitle, TopScore, …
@theburningmonk
SELECT * FROM GameScore
pAsterisk
*
pAttributeNames
UserId, GameTitle, TopScore, …
let pAttribute =
pAsterisk <|> pAttributeNames
@theburningmonk
SELECT * FROM GameScore
pAsterisk
*
pAttributeNames
UserId, GameTitle, TopScore, …
let pAttribute =
pAsterisk <|> pAttributeNames
@theburningmonk
SELECT * FROM GameScore
pAsterisk
*
pAttributeNames
UserId, GameTitle, TopScore, …
choice
pAttribute
@theburningmonk
SELECT * FROM GameScore
pAttribute
@theburningmonk
SELECT * FROM GameScore
pAttribute pTableNamepFrompSelect
@theburningmonk
SELECT * FROM GameScore
pAttribute pTableNamepFrompSelect
let pQuery =
tuple4 pSelect pAttribute
pFrom pTableName
|>> (fun (_, attributes, _, table) ->
{ Attributes = attributes
Table = table })
@theburningmonk
SELECT * FROM GameScore
pAttribute pTableNamepFrompSelect
let pQuery =
tuple4 pSelect pAttribute
pFrom pTableName
|>> (fun (_, attributes, _, table) ->
{ Attributes = attributes
Table = table })
@theburningmonk
SELECT * FROM GameScore
pAttribute pTableNamepFrompSelect
let pQuery =
tuple4 pSelect pAttribute
pFrom pTableName
|>> (fun (_, attributes, _, table) ->
{ Attributes = attributes
Table = table })
@theburningmonk
SELECT * FROM GameScore
pAttribute pTableNamepFrompSelect
let pQuery =
tuple4 pSelect pAttribute
pFrom pTableName
|>> (fun (_, attributes, _, table) ->
{ Attributes = attributes
Table = table })
Query
@theburningmonk
SELECT * FROM GameScore
pAttribute pTableNamepFrompSelect
tuple4
pQuery
@theburningmonk
@theburningmonk
< 50 lines of code
@theburningmonk
Amazing
F# + FParsec
=
@theburningmonk
Recap
@theburningmonk
@theburningmonk
select GameTitle, UserId, TopScore
from GameScores
where GameTitle = “Starship X”
and TopScore >= 1000
order desc
limit 3
with (NoConsistentRead, Index(GameTitleIndex, true))
@theburningmonk
Amazon
DynamoDB
Amazon
SimpleWorkflow
Amazon
CloudWatch
CASE STUDY
@theburningmonk
@theburningmonk
Decision Worker
@theburningmonk
Decision Worker
Poll
@theburningmonk
Decision Worker
Decision
Task
@theburningmonk
Decision WorkerDecide
@theburningmonk
Activity Worker
Decision Worker
@theburningmonk
Activity Worker
Decision Worker
Poll
@theburningmonk
Activity Worker
Decision Worker
Activity
Task
@theburningmonk
Activity Worker
Decision Worker
Complete
@theburningmonk
Workers can run from
anywhere
@theburningmonk
@theburningmonk
input = “Yan”
result = “Hello Yan!”
Start
Finish
Activity
@theburningmonk
image by Ryan Hageman license : https://creativecommons.org/licenses/by-sa/2.0/
SWF-based Application
API
Heartbeats
Error
Handling
Polling
API
Activity
Worker
Decision
Worker
Heartbeats
Error
Handling
Polling
API
Activity
Worker
Decision
Worker
Heartbeats
Error
Handling
Polling
API
Boilerplate
– Kris Jordan
“Good simplicity is less with
leverage, not less with less.
Good simplicity is complexity
disguised, not complexity denied.”
http://bit.ly/1pOLeKl
Tame cloud complexity with F#-powered DSLs
Start
Finish
Activity
?
@theburningmonk
the workflow is
implied by decision
worker logic…
@theburningmonk
instead..
@theburningmonk
the workflow should
drive decision
worker logic
@theburningmonk
the workflow should
driveautomate
decision worker logic
Amazon
.SimpleWorkflow
.Extensions
github.com/fsprojects/Amazon.SimpleWorkflow.Extensions
@theburningmonk
GOAL
Remove
boilerplates
@theburningmonk
GOAL
Code that matches
the way you think
@theburningmonk
Start
Finish
Activity
Tame cloud complexity with F#-powered DSLs
Tame cloud complexity with F#-powered DSLs
Start
Finish
Activity
Tame cloud complexity with F#-powered DSLs
@theburningmonk
Workflows can be
nested
Tame cloud complexity with F#-powered DSLs
Tame cloud complexity with F#-powered DSLs
input
result
Tame cloud complexity with F#-powered DSLs
@theburningmonk
Recap
Activity
Worker
Decision
Worker
Heartbeats
Error
Handling
Polling
API
Tame cloud complexity with F#-powered DSLs
@theburningmonk
Amazon
DynamoDB
Amazon
SimpleW
Amazon
CloudWatch
CASE STUDY
@theburningmonk
@theburningmonk
@theburningmonk
@theburningmonk
wanna find
correlations?
@theburningmonk
wanna find
correlations?
you can DIY it!
;-)
@theburningmonk
“what latencies spiked
at the same time as
payment service?”
Amazon
.CloudWatch
.Selector
github.com/fsprojects/Amazon.CloudWatch.Selector
@theburningmonk
Find metrics whose 5
min average exceeded
1 second during last
12 hours
@theburningmonk
cloudWatch.Select(
unitIs “milliseconds” +
average (>) 1000.0
@ last 12 hours
|> intervalOf 5 minutes)
@theburningmonk
cloudWatch.Select(“
unitIs ‘milliseconds’ and
average > 1000.0
duringLast 12 hours
at intervalOf 5 minutes”)
@theburningmonk
“did any cache
nodes’ CPU spike
yesterday?”
@theburningmonk
cloudWatch.Select(
namespaceLike “elasticache” +
nameLike “cpu” +
max (>) 80.0
@ last 24 hours
|> intervalOf 15 minutes)
@theburningmonk
cloudWatch.Select(
namespaceLike “elasticache” +
nameLike “cpu” +
max (>) 80.0
@ last 24 hours
|> intervalOf 15 minutes)
Regex
@theburningmonk
@theburningmonk
@theburningmonk
@theburningmonk
namespaceIs ‘JustEat’ and
nameLike ‘cpu’ and
unitIs ‘milliseconds’ and
average > 1000.0
duringLast 12 hours
at intervalOf 5 minutes
@theburningmonk
namespaceIs ‘JustEat’ and
nameLike ‘cpu’ and
unitIs ‘milliseconds’ and
average > 1000.0
duringLast 12 hours
at intervalOf 5 minutes
Filters
@theburningmonk
namespaceIs ‘JustEat’ and
nameLike ‘cpu’ and
unitIs ‘milliseconds’ and
average > 1000.0
duringLast 12 hours
at intervalOf 5 minutes
TimeFrame
@theburningmonk
namespaceIs ‘JustEat’ and
nameLike ‘cpu’ and
unitIs ‘milliseconds’ and
average > 1000.0
duringLast 12 hours
at intervalOf 5 minutes
Period
@theburningmonk
type Query =
{
Filter : Filter
TimeFrame : TimeFrame
Period : Period option
}
@theburningmonk
Query
Internal
DSL
External
DSL
@theburningmonk
namespaceIs ‘JustEat’ and
nameLike ‘cpu’ and
unitIs ‘milliseconds’ and
average > 1000.0
duringLast 12 hours
at intervalOf 5 minutes
@theburningmonk
type MetricTerm = Namespace | Name
type Filter =
| MetricFilter of MetricTerm * (string -> bool)
@theburningmonk
namespaceIs ‘JustEat’ and
nameLike ‘cpu’ and
unitIs ‘milliseconds’ and
average > 1000.0
duringLast 12 hours
at intervalOf 5 minutes
@theburningmonk
type MetricTerm = Namespace | Name
type Unit = | Unit
type Filter =
| MetricFilter of MetricTerm * (string -> bool)
| UnitFilter of Unit * (string -> bool)
@theburningmonk
namespaceIs ‘JustEat’ and
nameLike ‘cpu’ and
unitIs ‘milliseconds’ and
average > 1000.0
duringLast 12 hours
at intervalOf 5 minutes
@theburningmonk
type MetricTerm = Namespace | Name
type Unit = | Unit
type StatsTerm =
| Average | Min | Max | Sum | SampleCount
type Filter =
| MetricFilter of MetricTerm * (string -> bool)
| UnitFilter of Unit * (string -> bool)
| StatsFilter of StatsTerm * (float -> bool)
@theburningmonk
namespaceIs ‘JustEat’ and
nameLike ‘cpu’ and
unitIs ‘milliseconds’ and
average > 1000.0
duringLast 12 hours
at intervalOf 5 minutes
@theburningmonk
type MetricTerm = Namespace | Name
type Unit = | Unit
type StatsTerm =
| Average | Min | Max | Sum | SampleCount
type Filter =
| MetricFilter of MetricTerm * (string -> bool)
| UnitFilter of Unit * (string -> bool)
| StatsFilter of StatsTerm * (float -> bool)
| CompositeFilter of Filter * Filter
@theburningmonk
namespaceIs ‘JustEat’ and
nameLike ‘cpu’ and
unitIs ‘milliseconds’ and
average > 1000.0
duringLast 12 hours
at intervalOf 5 minutes
@theburningmonk
type TimeFrame =
| Last of TimeSpan
| Since of DateTime
| Between of DateTime * DateTime
@theburningmonk
namespaceIs ‘JustEat’ and
nameLike ‘cpu’ and
unitIs ‘milliseconds’ and
average > 1000.0
duringLast 12 hours
at intervalOf 5 minutes
@theburningmonk
type Period = | Period of TimeSpan
@theburningmonk
Active Patterns
a primer on
@theburningmonk
allow patterns to be
abstracted away into
named functions
@theburningmonk
Single-Case Patterns
@theburningmonk
let (|Float|) input =
match Double.TryParse input with
| true, n -> n
| _ ->
failwithf “not a float [%s]” input
@theburningmonk
let (|Float|) input =
match Double.TryParse input with
| true, n -> n
| _ ->
failwithf “not a float [%s]” input
@theburningmonk
let (|Float|) input =
match Double.TryParse input with
| true, n -> n
| _ ->
failwithf “not a float [%s]” input
Float : string -> float
@theburningmonk
match someString with
| Float 42.0 -> “ftw”
| Float 11.0 -> “palprime”
| Float x -> sprintf “just %f” x
@theburningmonk
let (|Float|) input =
match Double.TryParse input with
| true, n -> n
match someString with
| Float 42.0 -> “ftw”
| Float 11.0 -> “palprime”
| Float x -> sprintf “just %f” x
@theburningmonk
let (|Float|) input =
match Double.TryParse input with
| true, n -> n
match someString with
| Float 42.0 -> “ftw”
| Float 11.0 -> “palprime”
| Float x -> sprintf “just %f” x
@theburningmonk
match someString with
| Float 42.0 -> “ftw”
| Float 11.0 -> “palprime”
| Float x -> sprintf “just %f” x
let (|Float|) input =
match Double.TryParse input with
| true, n -> n
@theburningmonk
match someString with
| Float 42.0 -> “ftw”
| Float 11.0 -> “palprime”
| Float x -> sprintf “just %f” x
let (|Float|) input =
match Double.TryParse input with
| true, n -> n
@theburningmonk
match “42” with
| Float 42.0 -> “ftw”
| Float 11.0 -> “palprime”
| Float x -> sprintf “just %f” x
@theburningmonk
match “boo” with
| Float 42.0 -> “ftw”
| Float 11.0 -> “palprime”
| Float x -> sprintf “just %f” x
Error!!!
@theburningmonk
Partial Patterns
@theburningmonk
let (|Float|_|) input =
match Double.TryParse input with
| true, n -> Some n
| _ -> None
@theburningmonk
let (|Float|_|) input =
match Double.TryParse input with
| true, n -> Some n
| _ -> None
Float : string -> float option
@theburningmonk
match “boo” with
| Float 42.0 -> “ftw”
| Float 11.0 -> “palprime”
| Float x -> sprintf “just %f” x
| _ -> “not a float”
@theburningmonk
match “boo” with
| Float 42.0 -> “ftw”
| Float 11.0 -> “palprime”
| Float x -> sprintf “just %f” x
| _ -> “not a float”
@theburningmonk
Multi-Case Patterns
@theburningmonk
let (|Prime|NotPrime|NaN|) input =
match Double.TryParse input with
| true, n when isPrime n -> Prime n
| true, n -> NotPrime n
| _ -> NaN
@theburningmonk
let (|Prime|NotPrime|NaN|) input =
match Double.TryParse input with
| true, n when isPrime n -> Prime n
| true, n -> NotPrime n
| _ -> NaN
@theburningmonk
let (|Prime|NotPrime|NaN|) input =
match Double.TryParse input with
| true, n when isPrime n -> Prime n
| true, n -> NotPrime n
| _ -> NaN
@theburningmonk
let (|Prime|NotPrime|NaN|) input =
match Double.TryParse input with
| true, n when isPrime n -> Prime n
| true, n -> NotPrime n
| _ -> NaN
@theburningmonk
let (|Prime|NotPrime|NaN|) input =
match Double.TryParse input with
| true, n when isPrime n -> Prime n
| true, n -> NotPrime n
| _ -> NaN
@theburningmonk
match someString with
| Prime n -> …
| NotPrime n -> …
| NaN -> …
@theburningmonk
match someString with
| Prime n & Float 11.0 -> …
| Prime n -> …
| Float 42.0 | Float 8.0 -> …
| NotPrime n -> …
| NaN -> …
@theburningmonk
match someString with
| Prime n & Float 11.0 -> …
| Prime n -> …
| Float 42.0 | Float 8.0 -> …
| NotPrime n -> …
| NaN -> …
@theburningmonk
match someString with
| Prime (IsPalindrome n) -> …
| Prime (IsEven n) -> …
| _ -> …
@theburningmonk
Tokenise
(string -> string list)
@theburningmonk
[
“namespaceIs”; “‘JustEat’”; “and”;
“nameLike”; “‘cpu’”; “and”;
…
]
@theburningmonk
Visual Studio time…
@theburningmonk
Parse Filter
(string list -> Filter * string list)
@theburningmonk
let rec loop acc = function
| NamespaceIs (filter, tl)
| NamespaceLike (filter, tl)
| NameIs (filter, tl) | NameLike (filter, tl)
| UnitIs (filter, tl)
| Average (filter, tl) | Sum (filter, tl)
| Min (filter, tl) | Max (filter, tl)
| SampleCount (filter, tl)
-> match tl with
| And tl -> loop (filter::acc) tl
| _ -> flatten (filter::acc), tl
| _ -> failwith “No filters?!?!?”
@theburningmonk
let (|NamespaceIs|_|) = function
| StringCI "NamespaceIs"::QuotedString ns::tl
-> (eqFilter MetricFilter Namespace ns, tl)
|> Some
| _ -> None
let (|NamespaceLike|_|) = function
| StringCI "NamespaceLike"::QuotedString pattern::tl
-> (regexFilter MetricFilter Namespace pattern, tl)
|> Some
| _ -> None
@theburningmonk
Parse
(string list -> Query)
@theburningmonk
let parse (input : string) =
input
|> tokenize
|> parseFilter
|> parseTimeFrame
|> parsePeriod
@theburningmonk
Amazing
F#
=
@theburningmonk
usable from anywhere you
can run F# code
e.g. F# REPL, executable, ..
Internal DSL
@theburningmonk
useful for building tools
e.g. CLI, …
External DSL
@theburningmonk
@theburningmonk
@theburningmonk
@theburningmonk
Recap
@theburningmonk
@theburningmonk
@theburningmonk
Amazon
DynamoDB
Amazon
SimpleWorkflow
Amazon
CloudWatch
CASE STUDY
@theburningmonk
theburningmonk.com
github.com/theburningmonk
is hiring :-)
http://tech.just-eat.com/jobs
1 of 204

Recommended

F# in the real world (NDC) by
F# in the real world (NDC)F# in the real world (NDC)
F# in the real world (NDC)Yan Cui
260.7K views222 slides
F# at GameSys by
F# at GameSysF# at GameSys
F# at GameSysYan Cui
5.1K views254 slides
F# in the real world (CodeMotion Dubai) by
F# in the real world (CodeMotion Dubai)F# in the real world (CodeMotion Dubai)
F# in the real world (CodeMotion Dubai)Yan Cui
2.9K views210 slides
Very basic functional design patterns by
Very basic functional design patternsVery basic functional design patterns
Very basic functional design patternsTomasz Kowal
820 views77 slides
Refactor like a boss by
Refactor like a bossRefactor like a boss
Refactor like a bossgsterndale
592 views71 slides
Investigating Python Wats by
Investigating Python WatsInvestigating Python Wats
Investigating Python WatsAmy Hanlon
182 views80 slides

More Related Content

What's hot

optim function by
optim functionoptim function
optim functionSupri Amir
1.4K views50 slides
Elixir & Phoenix – fast, concurrent and explicit by
Elixir & Phoenix – fast, concurrent and explicitElixir & Phoenix – fast, concurrent and explicit
Elixir & Phoenix – fast, concurrent and explicitTobias Pfeiffer
5.8K views85 slides
Elixir & Phoenix – fast, concurrent and explicit by
Elixir & Phoenix – fast, concurrent and explicitElixir & Phoenix – fast, concurrent and explicit
Elixir & Phoenix – fast, concurrent and explicitTobias Pfeiffer
2.1K views83 slides
Introducción rápida a SQL by
Introducción rápida a SQLIntroducción rápida a SQL
Introducción rápida a SQLCarlos Hernando
739 views55 slides
Best Python Assignment Help by
Best Python Assignment HelpBest Python Assignment Help
Best Python Assignment HelpPython Homework Help
50 views16 slides
2 BytesC++ course_2014_c2_ flow of control by
2 BytesC++ course_2014_c2_ flow of control 2 BytesC++ course_2014_c2_ flow of control
2 BytesC++ course_2014_c2_ flow of control kinan keshkeh
173 views88 slides

What's hot(20)

optim function by Supri Amir
optim functionoptim function
optim function
Supri Amir1.4K views
Elixir & Phoenix – fast, concurrent and explicit by Tobias Pfeiffer
Elixir & Phoenix – fast, concurrent and explicitElixir & Phoenix – fast, concurrent and explicit
Elixir & Phoenix – fast, concurrent and explicit
Tobias Pfeiffer5.8K views
Elixir & Phoenix – fast, concurrent and explicit by Tobias Pfeiffer
Elixir & Phoenix – fast, concurrent and explicitElixir & Phoenix – fast, concurrent and explicit
Elixir & Phoenix – fast, concurrent and explicit
Tobias Pfeiffer2.1K views
2 BytesC++ course_2014_c2_ flow of control by kinan keshkeh
2 BytesC++ course_2014_c2_ flow of control 2 BytesC++ course_2014_c2_ flow of control
2 BytesC++ course_2014_c2_ flow of control
kinan keshkeh173 views
Turn Hours into Seconds - Concurrent event processing in Elixir using Flow by Emil Soman
Turn Hours into Seconds - Concurrent event processing in Elixir using FlowTurn Hours into Seconds - Concurrent event processing in Elixir using Flow
Turn Hours into Seconds - Concurrent event processing in Elixir using Flow
Emil Soman146 views
RxSwift 시작하기 by Suyeol Jeon
RxSwift 시작하기RxSwift 시작하기
RxSwift 시작하기
Suyeol Jeon3.6K views
A little exercise with clojure macro by Zehua Liu
A little exercise with clojure macroA little exercise with clojure macro
A little exercise with clojure macro
Zehua Liu1.3K views
How fast ist it really? Benchmarking in practice by Tobias Pfeiffer
How fast ist it really? Benchmarking in practiceHow fast ist it really? Benchmarking in practice
How fast ist it really? Benchmarking in practice
Tobias Pfeiffer4.7K views
Swift 함수 커링 사용하기 by 진성 오
Swift 함수 커링 사용하기Swift 함수 커링 사용하기
Swift 함수 커링 사용하기
진성 오2.3K views
A swift introduction to Swift by Giordano Scalzo
A swift introduction to SwiftA swift introduction to Swift
A swift introduction to Swift
Giordano Scalzo104.8K views
Nik Graf - Get started with Reason and ReasonReact by OdessaJS Conf
Nik Graf - Get started with Reason and ReasonReactNik Graf - Get started with Reason and ReasonReact
Nik Graf - Get started with Reason and ReasonReact
OdessaJS Conf152 views
To be Continued - multithreading with Project Loom and Kotlin's Coroutines by Artur Skowroński
To be Continued - multithreading with Project Loom and Kotlin's CoroutinesTo be Continued - multithreading with Project Loom and Kotlin's Coroutines
To be Continued - multithreading with Project Loom and Kotlin's Coroutines
Artur Skowroński167 views
The Ring programming language version 1.8 book - Part 96 of 202 by Mahmoud Samir Fayed
The Ring programming language version 1.8 book - Part 96 of 202The Ring programming language version 1.8 book - Part 96 of 202
The Ring programming language version 1.8 book - Part 96 of 202

Similar to Tame cloud complexity with F#-powered DSLs

Tame Cloud Complex with F# powered DSLs by
Tame Cloud Complex with F# powered DSLsTame Cloud Complex with F# powered DSLs
Tame Cloud Complex with F# powered DSLsYan Cui
818 views215 slides
Tame cloud complexity with F# powered DSLs (build stuff) by
Tame cloud complexity with F# powered DSLs (build stuff)Tame cloud complexity with F# powered DSLs (build stuff)
Tame cloud complexity with F# powered DSLs (build stuff)Yan Cui
652 views218 slides
NoSQL Cassandra Talk for Seattle Tech Startups 3-10-10 by
NoSQL Cassandra Talk for Seattle Tech Startups 3-10-10NoSQL Cassandra Talk for Seattle Tech Startups 3-10-10
NoSQL Cassandra Talk for Seattle Tech Startups 3-10-10egpeters
2K views18 slides
NoSQL Cassandra Talk for Seattle Tech Startups 3-10-10 by
NoSQL Cassandra Talk for Seattle Tech Startups 3-10-10NoSQL Cassandra Talk for Seattle Tech Startups 3-10-10
NoSQL Cassandra Talk for Seattle Tech Startups 3-10-10egpeters
534 views18 slides
Cassandra Seattle Tech Startups 3-10-10 by
Cassandra Seattle Tech Startups 3-10-10Cassandra Seattle Tech Startups 3-10-10
Cassandra Seattle Tech Startups 3-10-10egpeters
487 views18 slides
Cassandra Seattle Tech Startups 3-10-10 by
Cassandra Seattle Tech Startups 3-10-10Cassandra Seattle Tech Startups 3-10-10
Cassandra Seattle Tech Startups 3-10-10egpeters
261 views18 slides

Similar to Tame cloud complexity with F#-powered DSLs(20)

Tame Cloud Complex with F# powered DSLs by Yan Cui
Tame Cloud Complex with F# powered DSLsTame Cloud Complex with F# powered DSLs
Tame Cloud Complex with F# powered DSLs
Yan Cui818 views
Tame cloud complexity with F# powered DSLs (build stuff) by Yan Cui
Tame cloud complexity with F# powered DSLs (build stuff)Tame cloud complexity with F# powered DSLs (build stuff)
Tame cloud complexity with F# powered DSLs (build stuff)
Yan Cui652 views
NoSQL Cassandra Talk for Seattle Tech Startups 3-10-10 by egpeters
NoSQL Cassandra Talk for Seattle Tech Startups 3-10-10NoSQL Cassandra Talk for Seattle Tech Startups 3-10-10
NoSQL Cassandra Talk for Seattle Tech Startups 3-10-10
egpeters2K views
NoSQL Cassandra Talk for Seattle Tech Startups 3-10-10 by egpeters
NoSQL Cassandra Talk for Seattle Tech Startups 3-10-10NoSQL Cassandra Talk for Seattle Tech Startups 3-10-10
NoSQL Cassandra Talk for Seattle Tech Startups 3-10-10
egpeters534 views
Cassandra Seattle Tech Startups 3-10-10 by egpeters
Cassandra Seattle Tech Startups 3-10-10Cassandra Seattle Tech Startups 3-10-10
Cassandra Seattle Tech Startups 3-10-10
egpeters487 views
Cassandra Seattle Tech Startups 3-10-10 by egpeters
Cassandra Seattle Tech Startups 3-10-10Cassandra Seattle Tech Startups 3-10-10
Cassandra Seattle Tech Startups 3-10-10
egpeters261 views
Specs Presentation by Synesso
Specs PresentationSpecs Presentation
Specs Presentation
Synesso951 views
... now write an interpreter (PHPem 2016) by James Titcumb
... now write an interpreter (PHPem 2016)... now write an interpreter (PHPem 2016)
... now write an interpreter (PHPem 2016)
James Titcumb297 views
SQL and PLSQL features for APEX Developers by Connor McDonald
SQL and PLSQL features for APEX DevelopersSQL and PLSQL features for APEX Developers
SQL and PLSQL features for APEX Developers
Connor McDonald95 views
Falcon初印象 by 勇浩 赖
Falcon初印象Falcon初印象
Falcon初印象
勇浩 赖1K views
James Thomas - Serverless Machine Learning With TensorFlow - Codemotion Berli... by Codemotion
James Thomas - Serverless Machine Learning With TensorFlow - Codemotion Berli...James Thomas - Serverless Machine Learning With TensorFlow - Codemotion Berli...
James Thomas - Serverless Machine Learning With TensorFlow - Codemotion Berli...
Codemotion308 views
Micro-ORM Introduction - Don't overcomplicate by Kiev ALT.NET
Micro-ORM Introduction - Don't overcomplicateMicro-ORM Introduction - Don't overcomplicate
Micro-ORM Introduction - Don't overcomplicate
Kiev ALT.NET4.5K views
What you forgot from your Computer Science Degree by Stephen Darlington
What you forgot from your Computer Science DegreeWhat you forgot from your Computer Science Degree
What you forgot from your Computer Science Degree
Stephen Darlington2.5K views
Beware the potholes on the road to serverless by Yan Cui
Beware the potholes on the road to serverlessBeware the potholes on the road to serverless
Beware the potholes on the road to serverless
Yan Cui865 views
Let's play a game with blackfire player by Marcin Czarnecki
Let's play a game with blackfire playerLet's play a game with blackfire player
Let's play a game with blackfire player
Marcin Czarnecki114 views
Perl6 Regexen: Reduce the line noise in your code. by Workhorse Computing
Perl6 Regexen: Reduce the line noise in your code.Perl6 Regexen: Reduce the line noise in your code.
Perl6 Regexen: Reduce the line noise in your code.
Itty bittypresentation lrug by Tom Crinson
Itty bittypresentation lrugItty bittypresentation lrug
Itty bittypresentation lrug
Tom Crinson383 views
Itty bittypresentation lrug by Skills Matter
Itty bittypresentation lrugItty bittypresentation lrug
Itty bittypresentation lrug
Skills Matter216 views
Build Lightweight Web Module by Morgan Cheng
Build Lightweight Web ModuleBuild Lightweight Web Module
Build Lightweight Web Module
Morgan Cheng450 views

More from Yan Cui

How to win the game of trade-offs by
How to win the game of trade-offsHow to win the game of trade-offs
How to win the game of trade-offsYan Cui
21 views84 slides
How to choose the right messaging service by
How to choose the right messaging serviceHow to choose the right messaging service
How to choose the right messaging serviceYan Cui
135 views118 slides
How to choose the right messaging service for your workload by
How to choose the right messaging service for your workloadHow to choose the right messaging service for your workload
How to choose the right messaging service for your workloadYan Cui
65 views113 slides
Patterns and practices for building resilient serverless applications.pdf by
Patterns and practices for building resilient serverless applications.pdfPatterns and practices for building resilient serverless applications.pdf
Patterns and practices for building resilient serverless applications.pdfYan Cui
170 views137 slides
Lambda and DynamoDB best practices by
Lambda and DynamoDB best practicesLambda and DynamoDB best practices
Lambda and DynamoDB best practicesYan Cui
817 views148 slides
Lessons from running AppSync in prod by
Lessons from running AppSync in prodLessons from running AppSync in prod
Lessons from running AppSync in prodYan Cui
1.1K views102 slides

More from Yan Cui(20)

How to win the game of trade-offs by Yan Cui
How to win the game of trade-offsHow to win the game of trade-offs
How to win the game of trade-offs
Yan Cui21 views
How to choose the right messaging service by Yan Cui
How to choose the right messaging serviceHow to choose the right messaging service
How to choose the right messaging service
Yan Cui135 views
How to choose the right messaging service for your workload by Yan Cui
How to choose the right messaging service for your workloadHow to choose the right messaging service for your workload
How to choose the right messaging service for your workload
Yan Cui65 views
Patterns and practices for building resilient serverless applications.pdf by Yan Cui
Patterns and practices for building resilient serverless applications.pdfPatterns and practices for building resilient serverless applications.pdf
Patterns and practices for building resilient serverless applications.pdf
Yan Cui170 views
Lambda and DynamoDB best practices by Yan Cui
Lambda and DynamoDB best practicesLambda and DynamoDB best practices
Lambda and DynamoDB best practices
Yan Cui817 views
Lessons from running AppSync in prod by Yan Cui
Lessons from running AppSync in prodLessons from running AppSync in prod
Lessons from running AppSync in prod
Yan Cui1.1K views
Serverless observability - a hero's perspective by Yan Cui
Serverless observability - a hero's perspectiveServerless observability - a hero's perspective
Serverless observability - a hero's perspective
Yan Cui385 views
How to ship customer value faster with step functions by Yan Cui
How to ship customer value faster with step functionsHow to ship customer value faster with step functions
How to ship customer value faster with step functions
Yan Cui652 views
How serverless changes the cost paradigm by Yan Cui
How serverless changes the cost paradigmHow serverless changes the cost paradigm
How serverless changes the cost paradigm
Yan Cui1.1K views
Why your next serverless project should use AWS AppSync by Yan Cui
Why your next serverless project should use AWS AppSyncWhy your next serverless project should use AWS AppSync
Why your next serverless project should use AWS AppSync
Yan Cui1.3K views
Build social network in 4 weeks by Yan Cui
Build social network in 4 weeksBuild social network in 4 weeks
Build social network in 4 weeks
Yan Cui642 views
Patterns and practices for building resilient serverless applications by Yan Cui
Patterns and practices for building resilient serverless applicationsPatterns and practices for building resilient serverless applications
Patterns and practices for building resilient serverless applications
Yan Cui393 views
How to bring chaos engineering to serverless by Yan Cui
How to bring chaos engineering to serverlessHow to bring chaos engineering to serverless
How to bring chaos engineering to serverless
Yan Cui456 views
Migrating existing monolith to serverless in 8 steps by Yan Cui
Migrating existing monolith to serverless in 8 stepsMigrating existing monolith to serverless in 8 steps
Migrating existing monolith to serverless in 8 steps
Yan Cui402 views
Building a social network in under 4 weeks with Serverless and GraphQL by Yan Cui
Building a social network in under 4 weeks with Serverless and GraphQLBuilding a social network in under 4 weeks with Serverless and GraphQL
Building a social network in under 4 weeks with Serverless and GraphQL
Yan Cui289 views
FinDev as a business advantage in the post covid19 economy by Yan Cui
FinDev as a business advantage in the post covid19 economyFinDev as a business advantage in the post covid19 economy
FinDev as a business advantage in the post covid19 economy
Yan Cui546 views
How to improve lambda cold starts by Yan Cui
How to improve lambda cold startsHow to improve lambda cold starts
How to improve lambda cold starts
Yan Cui867 views
What can you do with lambda in 2020 by Yan Cui
What can you do with lambda in 2020What can you do with lambda in 2020
What can you do with lambda in 2020
Yan Cui1K views
A chaos experiment a day, keeping the outage away by Yan Cui
A chaos experiment a day, keeping the outage awayA chaos experiment a day, keeping the outage away
A chaos experiment a day, keeping the outage away
Yan Cui385 views
How to debug slow lambda response times by Yan Cui
How to debug slow lambda response timesHow to debug slow lambda response times
How to debug slow lambda response times
Yan Cui317 views

Recently uploaded

Piloting & Scaling Successfully With Microsoft Viva by
Piloting & Scaling Successfully With Microsoft VivaPiloting & Scaling Successfully With Microsoft Viva
Piloting & Scaling Successfully With Microsoft VivaRichard Harbridge
12 views160 slides
GDG Cloud Southlake 28 Brad Taylor and Shawn Augenstein Old Problems in the N... by
GDG Cloud Southlake 28 Brad Taylor and Shawn Augenstein Old Problems in the N...GDG Cloud Southlake 28 Brad Taylor and Shawn Augenstein Old Problems in the N...
GDG Cloud Southlake 28 Brad Taylor and Shawn Augenstein Old Problems in the N...James Anderson
85 views32 slides
Igniting Next Level Productivity with AI-Infused Data Integration Workflows by
Igniting Next Level Productivity with AI-Infused Data Integration Workflows Igniting Next Level Productivity with AI-Infused Data Integration Workflows
Igniting Next Level Productivity with AI-Infused Data Integration Workflows Safe Software
263 views86 slides
STKI Israeli Market Study 2023 corrected forecast 2023_24 v3.pdf by
STKI Israeli Market Study 2023   corrected forecast 2023_24 v3.pdfSTKI Israeli Market Study 2023   corrected forecast 2023_24 v3.pdf
STKI Israeli Market Study 2023 corrected forecast 2023_24 v3.pdfDr. Jimmy Schwarzkopf
19 views29 slides
Voice Logger - Telephony Integration Solution at Aegis by
Voice Logger - Telephony Integration Solution at AegisVoice Logger - Telephony Integration Solution at Aegis
Voice Logger - Telephony Integration Solution at AegisNirmal Sharma
39 views1 slide
【USB韌體設計課程】精選講義節錄-USB的列舉過程_艾鍗學院 by
【USB韌體設計課程】精選講義節錄-USB的列舉過程_艾鍗學院【USB韌體設計課程】精選講義節錄-USB的列舉過程_艾鍗學院
【USB韌體設計課程】精選講義節錄-USB的列舉過程_艾鍗學院IttrainingIttraining
52 views8 slides

Recently uploaded(20)

Piloting & Scaling Successfully With Microsoft Viva by Richard Harbridge
Piloting & Scaling Successfully With Microsoft VivaPiloting & Scaling Successfully With Microsoft Viva
Piloting & Scaling Successfully With Microsoft Viva
GDG Cloud Southlake 28 Brad Taylor and Shawn Augenstein Old Problems in the N... by James Anderson
GDG Cloud Southlake 28 Brad Taylor and Shawn Augenstein Old Problems in the N...GDG Cloud Southlake 28 Brad Taylor and Shawn Augenstein Old Problems in the N...
GDG Cloud Southlake 28 Brad Taylor and Shawn Augenstein Old Problems in the N...
James Anderson85 views
Igniting Next Level Productivity with AI-Infused Data Integration Workflows by Safe Software
Igniting Next Level Productivity with AI-Infused Data Integration Workflows Igniting Next Level Productivity with AI-Infused Data Integration Workflows
Igniting Next Level Productivity with AI-Infused Data Integration Workflows
Safe Software263 views
STKI Israeli Market Study 2023 corrected forecast 2023_24 v3.pdf by Dr. Jimmy Schwarzkopf
STKI Israeli Market Study 2023   corrected forecast 2023_24 v3.pdfSTKI Israeli Market Study 2023   corrected forecast 2023_24 v3.pdf
STKI Israeli Market Study 2023 corrected forecast 2023_24 v3.pdf
Voice Logger - Telephony Integration Solution at Aegis by Nirmal Sharma
Voice Logger - Telephony Integration Solution at AegisVoice Logger - Telephony Integration Solution at Aegis
Voice Logger - Telephony Integration Solution at Aegis
Nirmal Sharma39 views
【USB韌體設計課程】精選講義節錄-USB的列舉過程_艾鍗學院 by IttrainingIttraining
【USB韌體設計課程】精選講義節錄-USB的列舉過程_艾鍗學院【USB韌體設計課程】精選講義節錄-USB的列舉過程_艾鍗學院
【USB韌體設計課程】精選講義節錄-USB的列舉過程_艾鍗學院
STPI OctaNE CoE Brochure.pdf by madhurjyapb
STPI OctaNE CoE Brochure.pdfSTPI OctaNE CoE Brochure.pdf
STPI OctaNE CoE Brochure.pdf
madhurjyapb14 views
TouchLog: Finger Micro Gesture Recognition Using Photo-Reflective Sensors by sugiuralab
TouchLog: Finger Micro Gesture Recognition  Using Photo-Reflective SensorsTouchLog: Finger Micro Gesture Recognition  Using Photo-Reflective Sensors
TouchLog: Finger Micro Gesture Recognition Using Photo-Reflective Sensors
sugiuralab19 views
Special_edition_innovator_2023.pdf by WillDavies22
Special_edition_innovator_2023.pdfSpecial_edition_innovator_2023.pdf
Special_edition_innovator_2023.pdf
WillDavies2217 views
Empathic Computing: Delivering the Potential of the Metaverse by Mark Billinghurst
Empathic Computing: Delivering  the Potential of the MetaverseEmpathic Computing: Delivering  the Potential of the Metaverse
Empathic Computing: Delivering the Potential of the Metaverse
Mark Billinghurst478 views

Tame cloud complexity with F#-powered DSLs

Editor's Notes

  1. good afternoon, and welcome to this talk on taming complex APIs with DSLs and how FSharp can help you build these DSLs easily.
  2. My name is Yan Cui and I often go by the online alias of ‘theburningmonk’
  3. I work for a company called Gamesys, we're based in central London and are one of the market leaders in the real-money gaming business. Me and my team focus on freemium games for a more social audience and as a backend developer, I have built the backend for a number of our social games on Facebook and mobile.
  4. Across our social games, we have around 1 million DAU and, 250 millions requests per day.
  5. Pretty much every user action in our games are recorded and analyzed, we capture around 2TB of data a month for analytics purpose alone, which doesn’t take into account the significant amount of data we generate and store to facilitate the actual gameplay.
  6. All our core game services are deployed to Amazon Web Services, and we make extensive use of its many services..
  7. which has given us first-hand experience of the different kind of…
  8. complexities that can arise from some of these APIs.
  9. Some complexities are visible, they tend to be the result of the inherent complexities with the operations that the API allows you to perform
  10. Whilst other APIs might appear to be simple at what it does but pushes the complexities to you instead. These complexities tend to surface only when you start working with the API.
  11. And sometimes it’s merely a case of an impedance mismatch between what the API designer think you’ll do and what you actually need from the API
  12. We’ll use three AWS APIs that we use regularly as case studies to illustrate the different types of complexities and how…
  13. ..F# can be an effective weapon in tackling these complexities by simplifying the task of creating both internal and external DSLs.
  14. We’ll start with DynamoDB…
  15. which is to this day still the fastest growing service that Amazon has and the DB of choice for our small team…
  16. It is a managed key-value store…
  17. with built-in redundancy and 9 9s guarantee…
  18. it has great performance thanks to the fact that it runs on SSD drives…
  19. It differs from other Amazon services in that it doesn’t operate with the usual pay-as-you-go model where you pay for actual amount of usage you have for any particular service…
  20. Instead, when you create a table, you specify the throughput you require, Amazon would reserve enough capacity to meet your requirements and you pay for the amount of resources Amazon has to reserve even if you don’t end up using all that throughput…
  21. Once created, you can still change the throughput of a table on the fly without impacting performance or needing downtime…
  22. and the amount you pay will be adjusted when you change the throughput settings.
  23. You can create a DB that can handle a million transactions/s with a few clicks but it will cost you dearly if you need that level of throughput 24/7. It does enable an interesting usage pattern though, I know of an advertising firm which sees little traffic all year round but gets 4 million hits/s during the Superbowl, they were able to bump their throughput requirements all the way up during the Superbowl and then change them back afterwards. They managed to get through the Superbowl for a few thousand bucks and didn’t need to create and maintain a super-scalable, but expensive infrastructure that goes unused 99% of the time.
  24. It is semi-schema’d which means the only schema info you need to provide is the shape of the data you are going to use as the key in your table…
  25. For every hash key, you can specify an additional range key which you can filter on when making a query against your data. Whilst you’ll most likely be doing simple CRUD operations against specific keys - like you do in a key-value store - DynamoDb supports a limited set of queriability.
  26. A query in DynamoDB must be provided with a hash key, accompanied with a set of filters against the range key. If you created a local secondary index on your table then you can also filter against the local secondary index instead. Speaking of indices…
  27. It supports two types of indices, local secondary index which gives you an alternative range key to query with. Whereas global secondary index effectively allows you to specify an alternative set of hash and range key for the whole table.
  28. In our example here, most of the time we’ll be accessing data for individual players, so we use the UserId as Hashkey, and allow us to query the player’s scores by the game Title. The local index allows us to also query, for a give user, his top score. The global index, allows us to use Game Title has the hash key, and the TopScore as the range key. Which makes it easy for us to do queries such as - give me the top 10 players for Donkey Kong by score.
  29. DynamoDB also supports full table scans, but these tend to eat up your provisioned throughput, and take a long time to complete. Whilst it’s running you’re also more likely to have your normal queries throttled by DynamoDB if the Scan takes up too much throughput.
  30. If you consult the documentation for the Query API, you quickly get the sense that it’s anything but straight forward
  31. As an example, take the table we saw earlier, if we were to ask for…
  32. The top 3 player in Starship X by their Top Score, we might write…
  33. this… which is a) a lot of code, and b) the code is not easily comprehensible because there are a lot of noise. wouldn’t it be nice if you can write something like…
  34. this instead? Which is far less code and more expressive of our intent.
  35. Since data access is such a common task, it makes sense for us to create a DAL layer that provides a nicer abstraction for our game developers to work with.
  36. It’s also a great place to implement Timeout, Circuit Breaker, Caching, and other useful patterns for better performance, resilience and fault tolerance
  37. However, it’s hard to abstract over a complex API like Query, so the game developers end up having to work with the request objects from the AWSSDK directly whenever they want to Query/Scan the table.
  38. Which means the low-level abstractions provided by the AWSSDK itself is now leaking into the abstraction above…
  39. which is why we created…
  40. DynamoDB.SQL The goals of the project include…
  41. Providing a DSL that hides the complexity of the query/scan API and to…
  42. prevent AWSSDK from leaking through to downstream code
  43. We wanted to create an external DSL that is SQL like, so that it’s familiar to most developers, esp someone new joining the team. To do so…
  44. we need to parse the query into an AST and then translate that AST into a request and execute it against DynamoDB
  45. To create this solution, we used FSharp with FParsec, which is a …
  46. Parser combinator library. It basically means that you create basic, simple parsers and then combine them using functions to create more complex parsers. The functions that combine parsers together are called combinators.
  47. Take this simple query for instance…
  48. We have SELECT and FROM as keywords, GameScore is the name of the table, and for attributes we can either use a wildcard to fetch all the attributes or provide the name of the attributes we want in a comma separated list.
  49. In the AST, we can represent the attributes as a FSharp…
  50. DU, which you can think of as an Enum where each name can be associated with an arbitrary data type.
  51. Here, we say that attributes can be either an Asterisk, or an array of strings
  52. As for the whole query, we need to know the attributes, the table name, and we can store them inside a FSharp…
  53. Record type, which is a lightweight data container where the fields are…
  54. immutable by default.
  55. To build a parser for this simple query, we first need a parser for the select keyword.
  56. in FParsec, you can use the skipStringCI function which…
  57. matches against the string “select” in a case insensitive way, and ignores it
  58. we can do the same with the FROM keyword
  59. for the table name..
  60. DynamoDB requires a table name to consist of only letters and digits, so…
  61. we create a predicate function by combining the isLetter and isDigit predicates built into FParsec, using…
  62. a custom combinator that combines two predicates using OR. Since isLetter and isDigit predicates work against individual characters in a string, so…
  63. to match a string that is NOT EMPTY, and consists of only letters and digits we use the FParsec combinator many1Satisfy and provide it with our isTableName predicate function
  64. This creates a parser for our table name
  65. For the attributes, we first need a parser for …
  66. asterisk, and the stringCIReturn function here differs from the skipStringCI function we saw earlier in that..
  67. when matched, it will return the specified value, in our case, when we match the * we will return the union case Asterisk
  68. to parse a comma separated list of attribute names, we need a parser for individual attribute names, similar to the table name parser we saw previously
  69. we will also need a parser for commas, which is another use of the skipStringCI function
  70. we can then combine them together using the sepBy1 combinator function to create a parser that…
  71. parses one or more attribute names separated by commas
  72. now that we have both parsers, time to combine them…
  73. with the choice combinator, which can be denoted as…
  74. this special operator, that says…
  75. an attribute can be either an asterisk or a comma separated list of attribute names
  76. we now have all…
  77. the parsers that we need, to put everything together…
  78. we use the tuple4 combinator which takes in..
  79. four parsers that need to match in the specified order, this creates a parser which outputs a tuple of 4 items. The output can then be forwarded using this…
  80. combinator operator to a function that maps the tuple into…
  81. the query type we defined earlier, hence creating a parser that returns an instance of the Query type
  82. You can then incrementally add on the WHERE, ORDER, LIMIT, and WITH clauses to the parser, all and all…
  83. the query syntax took less than…
  84. 50 LOC, which just goes to show how…
  85. much productivity you get from using F# and FParsec.
  86. So to recap, we looked at…
  87. DynamoDB’s query operation as an example of a complex API that you can simplify with an external DSL…
  88. and how such a DSL can be easily implemented with F# and FParsec
  89. Next, let’s look at Amazon SimpleWorkflow, which presented a whole different kind of complexity.
  90. Simple Workflow is an orchestration service that manages the state of your workflow and provides the reporting, heartbeat and retry capabilities.
  91. To build an application on top of SWF, you need to implement a decision worker, which is responsible for…
  92. pulling tasks from the service…
  93. the decision task it gets back contains information about the current state of the workflow and the events that have occurred so far. Based on this, the worker needs to…
  94. decide what happens next in the workflow, whether the it should be completed, cancelled, or some more work needs to be scheduled. And when you need to do some work, you need an…
  95. activity worker, which also…
  96. pull for tasks…
  97. and the activity tasks it receives contains payloads for the activity that it needs to perform, be it to process some credit card payment or to encode some video, and when it’s done it…
  98. needs to report back to the service that the activity was completes. This triggers another decision task to be made available to the decision worker to process in order to decide what to do next.
  99. Your decision and activity workers doesn’t have to run from EC2, instead it can run from anywhere where you can access the SWF service.
  100. Everything that happens in a workflow execution is recorded and can be viewed from the management console, the input and result of every action, when it started and when it ended. You can retry failed executions, and workflows can be nested.
  101. As a hello world example, imagine we have a workflow with only one activity - which takes an input string and returns a string as result, here is what the implementation look like…
  102. which is pretty insane how much work was required, and we haven’t even implemented common concerns such as error handling and heartbeats. And the only thing in this craziness that is even relevant to what I wanted to get out of the service is…
  103. this line at the top…
  104. there’s gotta be a better way. Writing a truck load of code should never be your first-solution to a problem, every line of code you write has a cost - from the time it takes to write it, and then subsequently reading it, plus the time it takes to comprehend it, and then maintaining it. And considering that by making it easier for you to write more code, you’ve also made it easier for you to create cost to your company, which has a non-zero probability of outweighing the value that line of code brings.
  105. Before we look at our interpretation of a better way is, let’s look at the problem more closely. For any application that is built on top of Amazon simple workflow…
  106. first, there’s the service API…
  107. then there’re the recurring concerns such as polling, heartbeats, health checks and error handling…
  108. Then finally you have the things that are specific to your workflow, the activity and decision workers. That’s a lot of things you need to implement just to get going, but really, the only thing that is unique to your workflow are the activities it needs to perform, and everything else is…
  109. just glorified boilerplate. So whilst Simple Workflow’s APIs are pretty simple, as a consumer of the API, there are a lot of complexities that have been pushed down to you. Which reminds me of a post that Kris Jordan wrote a while back called ‘the complexity of simplicity’…
  110. where he said that…(read)…he went on to expand on this point by talking about how Google hides all the complexities around internet search and gives its users a simple UI with which they can do more with less. The guys at Stripe has done a very similar thing with taking credit card payments.
  111. It’s a really good read, here’s a bit.ly link to the post for you to have a look later on.
  112. in the decision worker implementation earlier, you know, the thing that orchestrates the workflow, plumbing aside, the highlighted block of code is responsible for scheduling the activity, and completing or failing the workflow after the activity either completes or fails. Looking at this block of code, I can’t easily reconstruct the mental model I had…
  113. of the workflow I set out to implement? The problem here is that…
  114. the workflow itself is never made explicit, but rather, implied by the logic that’s coded up in the decision worker, which is…
  115. far from ideal. Instead, the mental model you have of a workflow should be…
  116. driving the decision worker logic, or better, it should…
  117. automate the decision worker logic…
  118. and that’s why we created…
  119. a simple workflow extensions library which aims to…
  120. remove the boilerplate code you need to write, and…
  121. allow you to write code that actually matches your mental model of workflows
  122. and with that, let’s have a look how we’d implement our hello world example using our DSL
  123. The DSL allows you to configure the Workflow and attach activities to it using a simple…
  124. arrow-like operator, and when you read this code, you just follow the arrow to see…
  125. what your workflow does
  126. which each activity you add, you need to provide some configuration values and a delegate function which will be called when an activity task is received. It also automatically registers the workflow and activities with the service, something that you’d have to do manually otherwise
  127. Simple workflow allows you to nest workflows, so you can start and wait for another workflow to complete before continuing on in the parent workflow, to do this in the DSL…
  128. you might write something along the lines of…
  129. notice how we created a child workflow here, with its own activties, and…
  130. simply added it to a parent workflow as if it’s another activity
  131. the DSL takes care of the propagation of input and results so that the result of the previous activity or child workflow is passed as the input for the next activity or workflow
  132. In case of exceptions, it also takes care of capturing exception details and reporting back to the service so that you can see what went wrong in the management console
  133. you can optionally specify how many times an activity or child workflow should be retried
  134. Now, to recap, we looked at…
  135. Simple Workflow and how its simple API ends up pushing a lot of the complexities to you, the consumer, and how we addressed these complexities with…
  136. a super-simple internal DSL in FSharp that consists of an operator, and two types, that…
  137. …removes the need for boilerplate, and lets you create workflows with code that visually informs of what the workflow is going to do. It takes care of the heavy lifting involved so that you can…
  138. focus on building things that actually add value to your customers
  139. Finally, let’s have a look at Amazon CloudWatch..
  140. CloudWatch is a monitoring service that lets you collect and monitor metrics, and set alarms on those metrics. You can monitor anything from CPU usage, to database latencies, as well as any custom metrics you track from your application.
  141. It also comes with a nice web UI for browsing your metrics and charting them. It’s an immensely important and useful service, but, there are shortcomings…
  142. Presumably for performance reasons, the web UI limits you to 200 metrics when browsing, beyond that you have to know what you’re looking for to find it, so discovery is out of the question, but more importantly…
  143. to find correlations between events such as latency spikes, you have to manually search for, and inspect every latency metric yourself…
  144. it’s DevOps where you do all the monkey work, and you can’t even easily answer simple questions such as…
  145. …(read)… driven by pain and anger we decided to automate with…
  146. Amazon.CloudWatch.Selector, which gives us the ability to express queries using both internal and external DSLs
  147. For instance, if we want to find latency metrics that, at some point in the last 12 hours, exceeded a 1s average…
  148. we can express our question using an internal DSL that is very humanly readable, or as an…
  149. external DSL that is very similar in syntax and identical in capability
  150. or, suppose you want to find out if any of your cache nodes had a CPU spike in the last 24 hours…
  151. you might try something that targets specific metrics, in our case, CPU metrics under the Amazon ElastiCache namespace, and…
  152. whenever you see function names that end with ‘like’ it means they support regex
  153. overall the syntax for the DSL is quite small…
  154. as you can see, the internal DSL is implemented in a handful of lines…
  155. the external DSL is slightly more involved, and this time around I chose to implement it using active patterns, which is a…
  156. language feature in FSharp that allows you to abstract away, and give names to patterns so that they can be easily reused in pattern matching…
  157. there’s the single-case patterns which is similar to a normal function, this pattern here…
  158. matches against a string parameter called input, and either returns a float, or errors…
  159. and we’ll give this pattern a name, enclosed in what we call the banana clip operators…
  160. this pattern has a type signature of taking in a string, and returning a float, to use this pattern you could apply it…
  161. in a match…with clause where you can stack them up, or even compose different patterns together…
  162. and notice that the return value of the pattern can be bound and used in the pattern matching too…
  163. but, if someone pass in a string that can’t be parsed to a float…
  164. then this code will error. But sometimes you need a pattern that doesn’t always match, which is when you can use a partial active pattern instead…
  165. So here we can rewrite the earlier pattern to return an option…
  166. so that if we’re able to parse the string as a float, we’ll…
  167. the value of the float as Some, which is equivalent to Haskell’s Just.
  168. And when we can’t parse it as a float, then we return None, which is again, equivalent to Haskell’s Nothing.
  169. So now we can rewrite our pattern matching code earlier with an additional clause that catches cases where the input string…
  170. cannot be parsed as a float.
  171. and finally, if the input value cannot be classified into just something or nothing, then you can use the multi-case active pattern…
  172. here we’ll try to parse the input string as a float and if the resulting float is a prime number, then return it as a Prime…
  173. alternatively, return it with as a NotPrime…
  174. and if we couldn’t even parse the input then return NaN instead.
  175. and from here you can use these patterns in pattern matching as you would with any other. You can even compose them together to make your pattern matching code even more expressive…
  176. and from here you can use these patterns in pattern matching as you would with any other. You can use active patterns to really easily create parsers, and compose them to make more interesting parsers like we did with FParsec…
  177. one of the most pleasing thing for me was that I was able to rely on the type system to guide me along the way and after I wrote both DSLs in a 2 hour sprint everything compiled and worked as expected at the first time of asking! Which gave me a beautiful feeling, and just what I needed at 4am!
  178. so even on its own using only vanilla functions and active patterns, F# is still awesome at building DSLs with
  179. for the internal DSL, it can be used from anywhere you can run F# code, including the REPL, or executables…
  180. the external DSL is mainly useful for building tools with, such as a CLI…
  181. which one is available via Chocolatey
  182. with the CLI tool, you can write your queries using the external DSL syntax and if we find any matching metrics, you can plot them on graphs for visual inspection
  183. To recap, we looked at CloudWatch and how its impedance mismatch renders routine post-mortem investigations into an exercise…
  184. of finding a needle in the haystack, and how simple internal and external DSLs make that needle…
  185. that much easier to find
  186. and that concludes our case studies, but F#’s awesomeness doesn’t end here
  187. in fact, we have used F# to build a number of tools and frameworks for working with specific Amazon services, including…
  188. a type provider for S3…
  189. Which allows you to interactively navigate, browse objects in your S3 account, without even leaving Visual Studio, and you get full intellisense support on the buckets and objects you find
  190. you can also search for items in S3, which is useful for buckets with large number of objects, our buckets for player states have millions of objects, each with up to thousands of versions.
  191. we also have Reacto-Kinesix, a framework…
  192. for building real-time stream processing applications with Amazon Kinesis service
  193. and Metricano, a library for…
  194. for collecting and publishing custom metrics to services such as Amazon CloudWatch, whereby you can…
  195. either track metrics manually from your code or to use PostSharp aspects to auto instrument your methods to record execution count and time metrics. These metrics can then be hooked up to publishers such as the builtin CloudWatch publisher or…
  196. one of your own.
  197. You can use the PostSharp aspects to target individual methods, or multi-cast to all methods in a class or assembly to apply tracking to all your service entry points, or database calls…
  198. and to publish the metrics to CloudWatch, you just need to call Publish.With with an instance of CloudWatchPublisher
  199. and then you’ll be able to see your metrics on the CloudWatch management console, simple!
  200. and with that, thank you very much for listening
  201. We have the first ever FSharp Exchange in London next April, so hope to see you there.
  202. If you have any questions for me after today, feel free to get in touch