This document summarizes how to build concurrent and parallel workflows using Elixir Flow. It shows how to transition from an eager sequential implementation to a lazy stream-based one, then introduce concurrency using GenStage. Finally, it demonstrates partitioning work into parallel tasks across multiple cores to achieve a 5x speedup. Key aspects covered include producers, consumers, and instrumentation of parallel jobs. Optimizing the partitioning strategy further improved performance by an additional 15x.

1. Elixir Flow:
How to build and tune concurrent,
parallel workflows
Luke Galea - CTO, Precision Nutrition

3. GenStage

4. Producer
defmodule GenstageExample.Producer do
use GenStage
def start_link(initial 0) do
GenStage.start_link(__MODULE__, initial,
name: __MODULE__)
end
def init(counter), do: {:producer, counter}
def handle_demand(demand, state) do
events = Enum.to_list(state..(state +
demand - 1))
{:noreply, events, state + demand}
end
end

5. Producer-Consumer
defmodule GenstageExample.ProducerConsumer do
use GenStage
require Integer
def start_link do
GenStage.start_link(__MODULE__, :state_doesnt_matter, name: __MODULE__)
end
def init(state) do
{:producer_consumer, state, subscribe_to: [GenstageExample.Producer]}
end
def handle_events(events, _from, state) do
numbers =
events
|> Enum.filter(&Integer.is_even/1)
{:noreply, numbers, state}
end
end

6. Consumer
defmodule GenstageExample.Consumer do
use GenStage
def start_link do
GenStage.start_link(__MODULE__, :state_doesnt_matter)
end
def init(state) do
{:consumer, state, subscribe_to:
[GenstageExample.ProducerConsumer]}
end
def handle_events(events, _from, state) do
for event <- events do
IO.inspect({self(), event, state})
end
# As a consumer we never emit events
{:noreply, [], state}
end
end

7. What is Flow?
Flow allows developers to express
computations on collections, similar
to the Enum and Stream modules,
although computations will be
executed in parallel using multiple
GenStages.

8. API

9. Case Study: Link Checker

10. Eager to Lazy to
Concurrent to Parallel

11. EagerHTTPoison.start
File.read!("element_content.csv")
|> NimbleCSV.RFC4180.parse_string
|> Enum.map(fn [id, markdown] -> [id, Earmark.as_html!(markdown)] end)
|> Enum.map( fn [id, html] -> [id, html |> Floki.find("a") |> Floki.attribute("href")] end)
|> Enum.reject(fn
[_, []] -> true
_ -> false
end)
|> Enum.flat_map( fn [id, links] -> links |> Enum.map(&([id, &1])) end)
|> Enum.reject( fn [_, url] ->
case URI.parse(url) do
%URI{scheme: nil} -> true
%URI{host: nil} -> true
%URI{path: nil} -> true
_ -> false
end
end)
|> Enum.reject( fn [_, url] ->
case HTTPoison.get(url) do
{:ok, %HTTPoison.Response{status_code: status_code}}
when status_code >= 200 and status_code <= 399 -> true
_ -> false
end
end)
|> Enum.to_list()
Time to run: 23mins 12secs

12. Eager
File.read!("element_content.csv")
|> CSV.parse_string
|> map(&render_markdown/1)
|> map(&find_links/1)
|> reject(&has_no_links?/1)
|> flat_map(&flatten_links/1)
|> reject(&invalid_link?/1)
|> map(&check_link/1)
|> reject(&is_valid_link/1)
|> to_list()

13. Eager Performance

14. Eager Performance

15. Instrumentation
http://teamon.eu/2016/measuring-visualizing-genstage-flow-with-gnuplot/

16. Instrumentation
Progress.start_link([:links_parsed, :url_checked])
File.read!("element_content.csv")
|> NimbleCSV.RFC4180.parse_string
|> Enum.map(&render_markdown/1)
|> Enum.map(&find_links/1)
|> Enum.reject(&has_no_links?/1)
|> Enum.flat_map( fn [id, links] -> links |> Enum.map(&([id, &1])) end)
|> Enum.reject(&invalid_link?/1)
|> Enum.map(fn x -> Progress.incr(:links_parsed); x end)
|> Enum.map(&check_link/1)
|> Enum.map(fn x -> Progress.incr(:url_checked); x end)
|> Enum.reject(fn [_, _, valid] -> valid end) #Discard valid
|> Enum.to_list()
Progress.done()

17. Eager
Performance

18. Lazy
HTTPoison.start
output = IO.stream(:stdio, :line)
File.stream!("element_content.csv")
|> CSV.parse_stream
|> Stream.map(&render_markdown/1)
|> Stream.map(&find_links/1)
|> Stream.reject(&has_no_links?/1)
|> Stream.flat_map(&flatten_links/1)
|> Stream.reject(&invalid_link?/1)
|> Stream.map(fn x -> Progress.incr(:links_parsed); x end)
|> Stream.map(&check_link/1)
|> Stream.map(fn x -> Progress.incr(:url_checked); x end)
|> Stream.reject(&is_valid_link/1)
|> Stream.into(output, &"#{inspect &1}n")
|> Stream.run
Time to run: 23mins 20secs
+8 secs

19. Lazy Performance

20. Lazy Performance

21. Flow

22. Concurrent?
File.stream!("element_content.csv")
|> CSV.parse_stream
|> Flow.from_enumerable(stages: 1)
|> Flow.map(&render_markdown/1)
|> Flow.map(&find_links/1)
|> Flow.reject(&has_no_links?/1)
|> Flow.flat_map(&flatten/links/1)
|> Flow.reject(&invalid_link?/1)
|> Flow.map(&check_link/1)
|> Flow.reject(&is_valid_link/1)
|> Enum.to_list()
Time to run: 22mins 18secs
-54 secs

24. FlowViz.start_link([:links_parsed, :url_checked])
FlowViz.plot()
File.stream!("element_content.csv")
|> CSV.parse_stream
|> Flow.from_enumerable(max_demand: 2, min_demand: 1)
|> Flow.map(&render_markdown/1)
|> Flow.map(&find_links/1)
|> Flow.reject(&has_no_links?/1)
|> Flow.flat_map(&flatten_links/1)
|> Flow.reject(&invalid_link?/1)
|> FlowViz.checkpoint(:links_parsed)
|> Flow.map(&check_link/1)
|> FlowViz.checkpoint(:url_checked)
|> Flow.reject(&is_valid_link/1)
|> Enum.to_list()

25. Demo

26. Parallel
File.stream!("element_content.csv")
|> NimbleCSV.RFC4180.parse_stream
|> Flow.from_enumerable(stages: System.schedulers_online)
|> Flow.map(&render_markdown/1)
|> Flow.map(&find_links/1)
|> Flow.reject(&has_no_links?/1)
|> Flow.flat_map( fn [id, links] -> links |> Enum.map(&([id, &1])) end)
|> Flow.reject(&invalid_link?/1)
|> Flow.each(fn _ -> FlowProgress.incr(:links_parsed) end)
|> Flow.map(&check_link/1)
|> Flow.each(fn _ -> FlowProgress.incr(:url_checked) end)
|> Flow.reject(fn [_, _, valid] -> valid end) #Discard valid
|> Enum.to_list()
Time to run: 4mins 21secs
5.3x speedup on 8 logical cores

27. Parallel Performance

28. Concurrent + Parallel
File.stream!("element_content.csv")
|> CSV.parse_stream
|> Flow.from_enumerable
|> Flow.map(&render_markdown/1)
|> Flow.map(&find_links/1)
|> Flow.reject(&has_no_links?/1)
|> Flow.flat_map(&flatten_links/1)
|> Flow.reject(&invalid_link?/1)
|> Flow.map(&check_link/1)
|> Flow.reject(fn [_, _, valid] -> valid end) #Discard valid
|> Enum.to_list()
CPU Bound
IO Bound
Partition
Partition

29. Partitioned
File.stream!("element_content.csv")
|> NimbleCSV.RFC4180.parse_stream
|> Flow.from_enumerable
|> Flow.map(&render_markdown/1)
|> Flow.map(&find_links/1)
|> Flow.reject(&has_no_links?/1)
|> Flow.flat_map( fn [id, links] -> links |> Enum.map(&([id, &1])) end)
|> Flow.reject(&invalid_link?/1)
|> Flow.each(fn _ -> FlowProgress.incr(:links_parsed) end)
|> Flow.partition
|> Flow.map(&check_link/1)
|> Flow.each(fn _ -> FlowProgress.incr(:url_checked) end)
|> Flow.reject(fn [_, _, valid] -> valid end) #Discard valid
|> Enum.to_list()
Time to run: 4mins 02secs
Only slightly faster than non-partitioned

30. What’s happening?

32. Optimized
File.stream!("element_content.csv")
|> NimbleCSV.RFC4180.parse_stream
|> Flow.from_enumerable
|> Flow.map(&render_markdown/1)
|> Flow.map(&find_links/1)
|> Flow.reject(&has_no_links?/1)
|> Flow.flat_map( fn [id, links] -> links |> Enum.map(&([id, &1])) end)
|> Flow.reject(&invalid_link?/1)
|> Flow.each(fn _ -> FlowProgress.incr(:links_parsed) end)
|> Flow.partition(stages: 50)
|> Flow.map(&check_link/1)
|> Flow.each(fn _ -> FlowProgress.incr(:url_checked) end)
|> Flow.reject(fn [_, _, valid] -> valid end) #Discard valid
|> Enum.to_list()
Time to run: 1mins 34secs
~15x faster!

35. Demand
Flow.partition(
max_demand: 2,
min_demand: 1,
stages: 50)
Time to run: 1mins 45secs