GenServer in action

GenServer in Action
Elixir Club 11

Ternopil, 2018

Process
iex(1)> spawn fn -> 1 + 1 end
#PID<0.90.0>
iex(2)> caller = self()
#PID<0.88.0>
iex(3)> spawn fn -> send caller, {:result, 1 + 1} end
#PID<0.93.0>
iex(4)> receive do
...(4)> {:result, result} -> result
...(4)> end
2

Process
iex(5)> spawn fn ->
...(5)> Process.sleep(6_000)
...(5)> send caller, {:result, 1 + 1}
...(5)> end
#PID<0.106.0>
iex(6)> receive do
...(6)> end
⏳⏳⏳⏳⏳⌛
2

Process
iex(7)> spawn fn ->
...(7)> Process.sleep(666_666_666)
...(7)> end
#PID<0.99.0>
iex(8)> spawn fn -> :nothing end
#PID<0.101.0>
iex(9)> spawn fn -> raise "error" end
#PID<0.103.0>
…
iex(11)> receive do
...(11)> end
⏳⏳⏳⏳⏳⏳⏳⏳⏳⏳⏳⏳⏳⏳⏳⏳⏳⏳⏳⏳⏳⏳⏳⏳⏳⏳⏳⏳⏳⏳⏳⏳⏳⏳⏳⏳⏳⏳⏳
💤 💤 💤 CTRL+C

Process: timeout
iex(1)> caller = self()
#PID<0.88.0>
iex(2)> spawn fn ->
...(2)> Process.sleep(666_666_666)
...(2)> end
#PID<0.94.0>
iex(3)> receive do
...(3)> {:result, result} -> {:ok, result}
...(3)> after
...(3)> 5_000 -> {:error, :timeout}
...(3)> end
⏳⏳⏳⏳⌛{:error, :timeout}

Process: message queue
iex(4)> send self(), {:message, 1}
iex(5)> send self(), {:another_message, 1}
iex(10)> Process.info(self(), :messages)
{:messages,
[
message: 1,
another_message: 1,
message: 2,
another_message: 2,
message: 3,
another_message: 3
]}

Process: message queue
{:messages,
[
message: 1,
another_message: 10,
message: 2,
another_message: 20,
message: 3,
another_message: 30
]}
iex(11)> receive do {:another_message, n} -> n end
10
iex(12)> receive do {:another_message, n} -> n end
20
{:messages, [message: 1, message: 2, message: 3, another_message: 30]}
iex(14)> receive do {:message, n} -> n end
1
{:messages, [message: 2, message: 3, another_message: 30]}
iex(16)> receive do {:something, n} -> n after 0 -> :no_matching_message end
:no_matching_message

Concurrent Tasks
Task.async / Task.await
ex(17)> task = Task.async(fn -> 1 + 1 end)
%Task{
owner: #PID<0.88.0>,
pid: #PID<0.106.0>,
ref: #Reference<0.1183799544.2034761732.30152>
}
iex(18)> Task.await(task)
2

Concurrent Tasks
Task.async / Task.await
iex(19)> tasks = Enum.map(1..10, &Task.async(fn ->
...(19)> Process.sleep(3_000)
...(19)> &1 * 100
...(19)> end))
[…]
iex(20)> Process.sleep(5_000)
:ok
iex(21)> Enum.map(tasks, &Task.await(&1))
[100, 200, 300, 400, 500, 600, 700, 800, 900, 1000]

Concurrent Tasks
Task.await timeout
await(task, timeout 5000)
…
A timeout, in milliseconds, can be given with default value
of 5000. If the timeout is exceeded, then the current
process will exit.
…
iex(22)> Task.async(fn -> Process.sleep(6_000) end) |> Task.await()
** (exit) exited in: Task.await(%Task{owner: #PID<0.88.0>, pid:
#PID<0.156.0>, ref: #Reference<0.891801449.431751175.151026>}, 5000)
** (EXIT) time out
(elixir) lib/task.ex:501: Task.await/2

Storing state
defmodule Storage do
def recursive(state) do
receive do
{:add, caller_pid, value} ->
new_state = state + value
send(caller_pid, {:result, new_state})
recursive(new_state)
end
end
end
iex(2)> storage_pid = spawn(fn -> Storage.recursive(0) end)
iex(3)> send(storage_pid, {:add, self(), 2})
iex(6)> flush()
{:result, 2}
{:result, 4}
{:result, 6}

Storing state
Agent
iex(7)> {:ok, pid} = Agent.start_link(fn -> 0 end)
{:ok, #PID<0.101.0>}
iex(8)> Agent.update(pid, fn state -> state + 1 end)
:ok
iex(9)> Agent.get(pid, fn state -> state end)
1
update(agent, fun, timeout 5000)
get(agent, module, fun, args, timeout 5000)

GenServer
A behaviour module for implementing the server of a client-
server relation.

A GenServer is a process like any other Elixir process and it
can be used to keep state, execute code asynchronously
and so on.

GenServer
defmodule Stack do
use GenServer
# Client
def start_link(default) do
GenServer.start_link(__MODULE__, default)
end
def push(pid, item) do
GenServer.cast(pid, {:push, item})
end
def pop(pid) do
GenServer.call(pid, :pop)
end
# Server (callbacks)
def handle_call(:pop, _from, [h | t]) do
{:reply, h, t}
end
def handle_cast({:push, item}, state) do
{:noreply, [item | state]}
end
end

GenServer
defmodule Stack do
use GenServer
# Client
end
end
def pop(pid) do
end
{:reply, h, t}
end
end
end
Executed in caller process

GenServer
defmodule Stack do
use GenServer
# Client
end
end
def pop(pid) do
end
{:reply, h, t}
end
end
end
Executed in server process

GenServer: timeout
call(server, request, timeout 5000)
end
def pop(pid) do
end
@impl true
Process.sleep(30_000)
{:reply, h, t}
end
@impl true
end

GenServer: timeout
iex(2)> {:ok, pid} = Stack.start_link([])
{:ok, #PID<0.95.0>}
iex(3)> Enum.each(1..5, &Stack.push(pid, &1))
:ok
ex(4)> Stack.pop(pid)
** (exit) exited in: GenServer.call(#PID<0.95.0>, :pop, 5000)
** (EXIT) time out
(elixir) lib/gen_server.ex:836: GenServer.call/3
iex(4)> Stack.pop(pid)
** (EXIT) time out
** (EXIT) time out
** (EXIT) time out
iex(4)> Process.info(pid, :messages)
{:messages,
[
{:"$gen_call", {#PID<0.88.0>, #Reference<0.3022523149.3093823489.83754>},
:pop},
:pop},
:pop}
]}

GenServer: timeout
ex(5)> Process.info(pid, :messages)
{:messages,
[
{:"$gen_call", {#PID<0.88.0>,
#Reference<0.3022523149.3093823489.83794>},
:pop}
]}
ex(6)> Process.info(pid, :messages)
{:messages, []}
iex(7)> flush()
{#Reference<0.3022523149.3093823489.83730>, 5}
{#Reference<0.3022523149.3093823489.83754>, 4}
{#Reference<0.3022523149.3093823489.83774>, 3}
{#Reference<0.3022523149.3093823489.83794>, 2}
:ok

Digital Signature
Microservice

Digital Signature Microservice
• Microservice - part of the Ukrainian eHealth system
infrastructure

• Digital signature validation

• Ukrainian standard DSTU 4145-2002

• Integration with proprietary C library via NIF

Digital Signature
Microservice
NIF
Proprietary
DSTU
4145-2002
LIB
Digitally signed
content
< 5s
Decoded content
Signer info
Signature validation
Certiﬁcates
OSCP Server

{:ok, result} =
DigitalSignatureLib.processPKCS7Data(data, certs, true)
result ==
%{
content: "{"hello": "world"}",
is_valid: true,
signer: %{
common_name: "XXX YYY ZZZ",
country_name: "UA",
drfo: "1234567890",
edrpou: "",
given_name: "XX YY",
locality_name: "М. КИЇВ",
organization_name: "ФІЗИЧНА ОСОБА",
organizational_unit_name: "",
state_or_province_name: "",
surname: "XX",
title: ""
},
validation_error_message: ""
}

Sounds easy?
DigitalSignatureLib.processPKCS7Data(…)
Elixir Process
Elixir Process
Elixir Process
Elixir Process
Incoming Requests
200 OK
200 OK
200 OK
200 OK
Responses

Problem 1: concurrency
tasks =
Enum.map(1..25, fn _ ->
Task.async(fn ->
DigitalSignatureLib.processPKCS7Data(data, certs, true)
end)
end)
Enum.map(tasks, fn task ->
{:ok, result} = Task.await(task, 30000)
IO.inspect(result)
end)

%{
content: ***
}
%{
content: ***
}
%{
content: ***
}
%{
content: ***
}
*** Error in `/usr/local/lib/erlang/erts-9.3.3/bin/beam.smp': double free or corruption (fasttop): 0x00007f88e401d060 ***
======= Backtrace: =========
/lib/x86_64-linux-gnu/libc.so.6(+0x70bfb)[0x7f8962452bfb]
/lib/x86_64-linux-gnu/libc.so.6(+0x76fc6)[0x7f8962458fc6]
/lib/x86_64-linux-gnu/libc.so.6(+0x7780e)[0x7f896245980e]
/usr/local/lib/libUACryptoQ.so.1(_ZN9DataBlockaSERKS_+0x3c)[0x7f88a7bd785c]
/usr/local/lib/libUACryptoQ.so.1(_ZN4DataaSERKS_+0x11)[0x7f88a7bd6771]
/usr/local/lib/libUACryptoQ.so.1(_ZN7DataSet3NewERK4Data+0x3c)[0x7f88a7bd81cc]
/usr/local/lib/libUACryptoQ.so.1(_ZNK3UAC16ObjectIdentifier10EncodeBodyEv+0x1fc)[0x7f88a7bf238c]
/usr/local/lib/libUACryptoQ.so.1(_ZNK3UAC5Token6EncodeEv+0x46)[0x7f88a7bef236]
/usr/local/lib/libUACryptoQ.so.1(_ZNK3UAC11ObjectToken10EncodeBodyEv+0x17)[0x7f88a7bef3c7]
/usr/local/lib/libUACryptoQ.so.1(_ZNK3UAC11Constructed10EncodeBodyEv+0x45)[0x7f88a7be97d5]
/usr/local/lib/libUACryptoQ.so.1(_ZNK3UAC11Constructed6EncodeEv+0x39)[0x7f88a7bec089]
/usr/local/lib/libUACryptoQ.so.1(_ZNK3UAC10CMSContent6EncodeEv+0x95)[0x7f88a7c0a2d5]
/usr/local/lib/libUACryptoQ.so.1(_ZNK3UAC11ObjectToken10EncodeBodyEv+0x3a)[0x7f88a7bef3ea]
/usr/local/lib/libUACryptoQ.so.1(_ZNK3UAC10CMSContent6EncodeEv+0x95)[0x7f88a7c0a2d5]
/usr/local/lib/libUACryptoQ.so.1(_ZNK3UAC17ObjectWithContent6EncodeEv+0x1b)[0x7f88a7be946b]
/usr/local/lib/libUACryptoQ.so.1(_ZN3UAC13CMSSignedData15VerifySignatureEPNS_11CertificateEP11_UAC_STREAM+0x9f5)[0x7f88a7c12965]
/usr/local/lib/libUACryptoQ.so.1(UAC_SignedDataVerify+0x2ba)[0x7f88a7c4ff1a]
/home/digital_signature.lib/_build/test/lib/digital_signature_lib/priv/digital_signature_lib_nif.so(Check+0x3c5)[0x7f891c0fc7f5]
/home/digital_signature.lib/_build/test/lib/digital_signature_lib/priv/digital_signature_lib_nif.so(+0x2bac)[0x7f891c0fbbac]
/usr/local/lib/erlang/erts-9.3.3/bin/beam.smp(erts_call_dirty_nif+0x19c)[0x5ce04c]
/usr/local/lib/erlang/erts-9.3.3/bin/beam.smp(erts_dirty_process_main+0x209)[0x445949]
/usr/local/lib/erlang/erts-9.3.3/bin/beam.smp[0x4f4a4d]
/usr/local/lib/erlang/erts-9.3.3/bin/beam.smp[0x675985]
/lib/x86_64-linux-gnu/libpthread.so.0(+0x7494)[0x7f8962990494]
/lib/x86_64-linux-gnu/libc.so.6(clone+0x3f)[0x7f89624caacf]
======= Memory map: ========
00400000-0073b000 r-xp 00000000 08:01 2376229 /usr/local/lib/erlang/erts-9.3.3/bin/beam.smp
0093a000-0093b000 r--p 0033a000 08:01 2376229 /usr/local/lib/erlang/erts-9.3.3/bin/beam.smp
0093b000-00956000 rw-p 0033b000 08:01 2376229 /usr/local/lib/erlang/erts-9.3.3/bin/beam.smp
00956000-0097d000 rw-p 00000000 00:00 0
0231e000-02349000 rw-p 00000000 00:00 0 [heap]

Problem 1 solution: Gen Server
...
use GenServer
# Callbacks
...
def handle_call({:process_signed_content, signed_content, check}, _from,
{certs_cache_ttl, certs}) do
check = unless is_boolean(check), do: true
processing_result = do_process_signed_content(signed_content, certs, check,
SignedData.new())
{:reply, processing_result, {certs_cache_ttl, certs}}
end
...
# Client
...
def process_signed_content(signed_content, check) do
GenServer.call(__MODULE__, {:process_signed_content, signed_content, check})
end

Problem 1 solution: Gen Server
NifService (GenServer)
NifService.process_signed_content(…)
Requests
Message queue
Responses

Problem 2: timeouts
Multiple concurrent
requests (processes)
2s
2s
2s
2s
Message queue
Sequential
responses
200 OK
2s
200 OK
4s
500 Error
5s
500 Error
5s
** (EXIT) time out
** (EXIT) time out

Problem 2: timeouts
Multiple concurrent
requests (processes)
2s
2s
Message queue
Sequential
responses
2s
2s
500 Error
5s
500 Error
5s

Problem 2 solution: architecture
DS
DS
DS
DS DS
DS
DS
DS
Load
Balancer
Reduce pressure on individual MS

Problem 2 solution: catch Exit
gen_server_timeout =
Confex.fetch_env(:digital_signature_api,
:nif_service_timeout)
try do
GenServer.call(__MODULE__, {:process_signed_content,
signed_content, check}, gen_server_timeout)
catch
:exit, {:timeout, error} ->
{:error, {:nif_service_timeout, error}}
end
end
424 Failed Dependency

Problem 3: message queue
Processed requests
2s
2s
2s
2s
Message queue
Sequential
responses
200 OK
2s
200 OK
4s
424
5s
424
5s
2s
2s
Expired requests
New requests

Problem 3 solution: pass
message expiration time
...
# Callbacks
def handle_call({:process_signed_content, signed_content, check, message_exp_time}, _from, certs_cache_ttl, certs})
do
processing_result =
if NaiveDateTime.diff(message_exp_time, NaiveDateTime.utc_now(), :millisecond) > 250 do
check = unless is_boolean(check), do: true
do_process_signed_content(signed_content, certs, check, SignedData.new())
else
{:error, {:nif_service_timeout, "messaqe queue timeout"}}
end
{:reply, processing_result, {certs_cache_ttl, certs}}
End
...
# Client
gen_server_timeout = Confex.fetch_env!(:digital_signature_api, :nif_service_timeout)
message_exp_time = NaiveDateTime.add(NaiveDateTime.utc_now(), gen_server_timeout, :millisecond)
try do
GenServer.call(__MODULE__, {:process_signed_content, signed_content, check, message_exp_time}, gen_server_timeout)
catch
:exit, {:timeout, error} ->
{:error, {:nif_service_timeout, error}}
end
end
...

Problem 3 solution: pass
message expiration time
…
Expired
Expired
Within threshold
Message queue
…

Problem 3 advanced solution:
QueueService
NifService
DigitalSignatureLib.
processPKCS7Data(…)
QueueService
:queue
Monitoring & Control

Memory management
Binary terms which are larger than 64 bytes are not stored in a process private
heap.

They are called Refc Binary (Reference Counted Binary) and are stored in a
large Shared Heap which is accessible by all processes who have the pointer
of that Refc Binaries.

That pointer is called ProcBin and is stored in a process private heap.

The GC for the shared heap is reference counting.

Collecting those Refc messages depends on collecting of all ProcBin objects
even ones that are inside the middleware process.

Unfortunately because ProcBins are just a pointer hence they are so cheap
and it could take so long to happen a GC inside the middleware process.

NifService
Memory management
Shared Heap
Refc Binary
Refc Binary
Refc Binary
Requests Responses
ProcBin
ProcBin
ProcBin
ProcBin
ProcBin
ProcBin
ProcBin
ProcBin
ProcBin
GC
GC
GC
:erlang.garbage_collect(self())
Processes

Garbage Collect
...
# Callbacks
def init(certs_cache_ttl) do
certs = CertAPI.get_certs_map()
Process.send_after(self(), :refresh, certs_cache_ttl)
{:ok, {certs_cache_ttl, certs}}
end
...
def handle_info(:refresh, {certs_cache_ttl, _certs}) do
certs = CertAPI.get_certs_map()
# GC
:erlang.garbage_collect(self())
Process.send_after(self(), :refresh, certs_cache_ttl)
{:noreply, {certs_cache_ttl, certs}}
end
...

Garbage Collect
handle_call(request, from, state)
{:reply, reply, new_state, timeout() | :hibernate}
Hibernating a GenServer causes garbage collection and leaves
a continuous heap that minimises the memory used by the
process
Returning {:reply, reply, new_state, timeout} is similar to
{:reply, reply, new_state} except handle_info(:timeout,
new_state) will be called after timeout milliseconds if no
messages are received

Useful links
Saša Jurić "Elixir in Action, Second Edition”
https://www.manning.com/books/elixir-in-action-second-edition

Andrea Leopardi - Concurrent and Resilient Connections to Outside the
BEAM (ElixirConfEU 2016)
https://www.youtube.com/watch?time_continue=1884&v=U1Ry7STEFiY

GenServer call time-outs
https://cultivatehq.com/posts/genserver-call-timeouts/

Elixir Memory - Not Quite Free
https://stephenbussey.com/2018/05/09/elixir-memory-not-quite-free.html

Erlang Garbage Collection Details and Why It Matters
https://hamidreza-s.github.io/erlang%20garbage%20collection%20memory%20layout%20soft%20realtime/
2015/08/24/erlang-garbage-collection-details-and-why-it-matters.html

Thank You
Yurii Bodarev

https://github.com/ybod

GenServer in action

More Related Content

What's hot

Similar to GenServer in action

Recently uploaded

GenServer in action