This talk was give for Elixir Montreal Meetup to talk about how the elixir formatter works and how it compiles the source code to generate a pretty format

1. Elixir Formatter Internals
How that works under the hood to
make your code pretty.
Pedro Medeiros

2. About Me
- 10 years of experience in programming
- Computer Scientist @ Universidade
Católica de Brasília
- Currently @ Shopify
- /pedrosnk
- /pesnk
- Has Deployed an Elixir App Before Phoenix
Fun Fact

3. Briefing
• Previous release of Elixir 1.5
• What is the Formatter
• Formatter internals
• Questions

4. State of Elixir 1.5
• Shipped features to help
developers and
documentation
• Breakpoints
• Exception Blame
• @impl

5. breakpoint
• A set of functions available on iex to help debug
• break!/2, break!/4, breaks/0, continue/0, open/0,
remove_breaks/0, remove_breaks/1, reset_break/1,
reset_break/3, respawn/0, whereiam/1

6. Iex breakpoint

7. Exception Blame
iex(1)> Access.fetch :foo, :bar
** (FunctionClauseError) no function clause matching in Access.fetch/2
The following arguments were given to Access.fetch/2:
# 1
:foo
# 2
:bar
Attempted function clauses (showing 5 out of 5):
def fetch(%struct{} = container, key)
def fetch(map, key) when is_map(map)
def fetch(list, key) when is_list(list) and is_atom(key)
def fetch(list, key) when is_list(list)
def fetch(nil, _key)
(elixir) lib/access.ex:261: Access.fetch/2

8. @impl
Add ability to mark which function
in a module is a implementation of
a call back.
defmodule Stack do
use GenServer
@impl GenServer
def handle_call(:pop, _from, [h | t]) do
{:reply, h, t}
end
@impl GenServer
def handle_cast({:push, item}, state) do
{:noreply, [item | state]}
end
end
defmodule MyApp do
@behaviour Plug
@impl Plug
def init(_opts) do
opts
end
@impl Plug
def call(conn, _opts) do
Plug.Conn.send_resp(conn, 200, "hello world")
end
end

10. Elixir Goals
Productivity
Maintainability
Reliability

11. Elixir Goals
bit.ly/elixirgoals2017

12. Formatter
• Formats your code to maintain a consistency style for all
elixir projects
• Community code standards and a common guide for
newcomers
• Maintain the same AST as the original code. Formatter is
not a Linter
• Built in inside mix

13. Run the formatter
defmodule MessedCode do
use GenServer
@default_list [
1,2,3,
4,5
]
def say_hello first_name, last_name do
"Hello #{first_name} #{last_name}"
end
@impl GenServer
def handle_call { :add_to_list, value}, _from,%{values: list,stage: stage }=_state do
stage = case stage do
"initial" -> "processing"
"progress" -> "final"
"final" -> "final"
_unkown -> "unkown"
end
{:reply, :ok,
%{value: list ++ [value],
stage: stage}}
end
end

14. Run the formatter
$ mix format _

15. Run the formatter
defmodule MessedCode do
use GenServer
@default_list [
1,
2,
3,
4,
5
]
def say_hello(first_name, last_name) do
"Hello #{first_name} #{last_name}"
end
@impl GenServer
def handle_call({:add_to_list, value}, _from, %{values: list, stage: stage} = _state) do
stage =
case stage do
"initial" -> "processing"
"progress" -> "final"
"final" -> "final"
_unkown -> "unkown"
end
{:reply, :ok, %{value: list ++ [value], stage: stage}}
end
end

16. What happened?

17. What happened?
defmodule Fac do
@spec fac(pos_integer) :: pos_integer
@doc "Compute factorial"
def fac 1 do; 1 ;end # stop condition
def fac n do
n * fac n - 1
end
end

18. What happened?> raw = File.read!("lib/fac.ex")
> formated = Code.format_string!(raw)
["defmodule", " ", "Fac", " do", "n ", "@spec", " ", "fac", "(", "",
"pos_integer", "", ")", " ::", " ", "pos_integer", "n ", "@doc", " ", """,
"Compute factorial", """, "n ", "# stop condition", "n ", …]
> formated |> Enum.join() |> IO.puts()
defmodule Fac do
@spec fac(pos_integer) :: pos_integer
@doc "Compute factorial"
# stop condition
def fac(1) do
1
end
def fac(n) do
n * fac(n - 1)
end
end

19. > raw = File.read!("lib/fac.ex")
> Code.Formatter.to_algebra(raw, opts)
Deeper inside
Code.format_string!
{:doc_group,
{:doc_cons, {:doc_string, "defmodule", 9},
{:doc_force,
{:doc_cons,
{:doc_cons,
{:doc_group,
{:doc_cons, {:doc_cons, " ", {:doc_nest, "Fac", :cursor, :break}}, " do"},
:self},
{:doc_nest,
{:doc_cons, :doc_nil,
{:doc_cons, :doc_line,
{:doc_group,
{:doc_force,
{:doc_cons,
{:doc_cons,
{:doc_cons,
{:doc_cons,
{:doc_group,
{:doc_cons, {:doc_string, "@spec", 5},
{:doc_group,
{:doc_cons,
{:doc_cons, " ",
{:doc_nest, {:doc_cons, {...}, ...}, :cursor, :break}},
:doc_nil}, :self}}, :self},
{:doc_cons, :doc_line,
{:doc_group,
{:doc_cons, {:doc_string, "@doc", 4},
{:doc_group, {:doc_cons, {:doc_cons, " ", {...}}, :doc_nil},
:self}}, :self}}},
{:doc_cons, :doc_line, {:doc_group, "# stop condition", :self}}},
{:doc_cons, :doc_line,
{:doc_group,
{:doc_cons,
{:doc_cons, {:doc_string, "def", 3},
{:doc_force,
{:doc_cons,
{:doc_cons, {:doc_group, {:doc_cons, ...}, :self},
{:doc_nest, {...}, ...}}, {:doc_cons, :doc_line, "end"}}}},
{:doc_cons, {:doc_collapse, 2}, {:doc_break, "", :strict}}},
:self}}},
{:doc_cons, :doc_line,
{:doc_group,
{:doc_cons, {:doc_break, "", :strict},
{:doc_cons, {:doc_string, "def", 3},
{:doc_force,
{:doc_cons,...

20. Deeper inside
Code.format_string!
Code.Formatter.to_algebra(raw)
:elixir.string_to_tokens/4
:elixir.tokens_to_quoted/3

21. Deeper inside
Code.format_string!
:elixir.string_to_tokens/4
defmodule Fac do
@spec fac(pos_integer) :: pos_integer
@doc "Compute factorial"
def fac 1 do; 1 ;end # stop condition
def fac n do
n * fac n - 1
end
end

22. Deeper inside
How an algebra is build for the formatter
:elixir.string_to_tokens
:elixir.tokens_to_quoted
[
{:identifier, {1, 1, nil}, :defmodule},
{:alias, {1, 11, nil}, :Fac},
{:do, {1, 15, nil}},
{:eol, {1, 17, 1}},
{:at_op, {2, 3, nil}, :@},
{:identifier, {2, 4, nil}, :spec},
{:paren_identifier, {2, 9, nil}, :fac},
{:"(", {2, 12, nil}},
{:identifier, {2, 13, nil}, :pos_integer},
{:")", {2, 24, nil}},
{:type_op, {2, 26, nil}, :::},
{:identifier, {2, 29, nil}, :pos_integer},
{:eol, {2, 40, 1}},
{:at_op, {3, 3, nil}, :@},
{:identifier, {3, 4, nil}, :doc},
{:bin_string, {3, 8, nil}, ["Compute factorial"]},
{:eol, {3, 27, 1}},
{:identifier, {4, 3, nil}, :def},
{:identifier, {4, 7, nil}, :fac},
{:int, {4, 11, 1}, '1'},
{:do, {4, 13, nil}},
{:";", {4, 15, 0}},
{:int, {4, 17, 1}, '1'},
{:";", {4, 19, 0}},
{:end, {4, 20, nil}},
{:eol, {4, 24, 1}},
{:identifier, {5, 3, nil}, :def},
{:identifier, {5, 7, nil}, :fac},
{:do_identifier, {5, 11, nil}, :n},
{:do, {5, 13, nil}},
{:eol, {5, 15, 1}},
{:identifier, {6, 5, nil}, :n},
{:mult_op, {6, 7, nil}, :*},
{:identifier, {6, 9, nil}, :fac},
{:identifier, {6, 13, nil}, :n},
{:dual_op, {6, 15, nil}, :-},
{:int, {6, 17, 1}, '1'},
{:eol, {6, 18, 1}},
{:end, {7, 3, nil}},
{:eol, {7, 6, 1}},
{:end, {8, 1, nil}},
{:eol, {8, 4, 1}}
]

23. Deeper inside
Code.format_string!
:elixir.tokens_to_quoted/3
[
{:identifier, {1, 1, nil}, :defmodule},
{:alias, {1, 11, nil}, :Fac},
{:do, {1, 15, nil}},
{:eol, {1, 17, 1}},
{:at_op, {2, 3, nil}, :@},
{:identifier, {2, 4, nil}, :spec},
{:paren_identifier, {2, 9, nil}, :fac},
{:"(", {2, 12, nil}},
{:identifier, {2, 13, nil}, :pos_integer},
{:")", {2, 24, nil}},
{:type_op, {2, 26, nil}, :::},
{:identifier, {2, 29, nil}, :pos_integer},
{:eol, {2, 40, 1}},
{:at_op, {3, 3, nil}, :@},
{:identifier, {3, 4, nil}, :doc}, ...

24. Deeper inside
Code.format_string!
{:defmodule, [no_parens: true, line: 1],
[
{:__aliases__, [line: 1], [:Fac]},
[
{{:__block__, [format: :block, end_line: 8, line: 1], [:do]},
{:__block__, [],
[
{:@, [line: 2],
[
{:spec, [no_parens: true, line: 2],
[
{:::, [line: 2],
[
{:fac, [end_line: 2, line: 2],
[{:pos_integer, [no_parens: true, line: 2], nil}]},
{:pos_integer, [no_parens: true, line: 2], nil}
]}
]}
]},
{:@, [newlines: 1, line: 3],
[
{:doc, [no_parens: true, line: 3],
[{:__block__, [format: :string, line: 3], ["Compute factorial"]}]}
]},
{:def, [newlines: 1, no_parens: true, line: 4],
[
{:fac, [no_parens: true, line: 4],
[{:__block__, [original: '1', line: 4], [1]}]},
[
{{:__block__, [format: :block, end_line: 4, line: 4], [:do]},
{:__block__, [original: '1', line: 4], [1]}}
]
]},
…

25. Deeper inside
Code.format_string!
quote do
defmodule Fac do
@spec fac(pos_integer) :: pos_integer
@doc "Compute factorial"
def fac 1 do; 1 ;end # stop condition
def fac n do
n * fac n - 1
end
end
end

26. {:defmodule, [context: Elixir, import: Kernel],
[
{:__aliases__, [alias: false], [:Fac]},
[
do: {:__block__, [],
[
{:@, [context: Elixir, import: Kernel],
[
{:spec, [context: Elixir],
[
{:::, [],
[
{:fac, [], [{:pos_integer, [], Elixir}]},
{:pos_integer, [], Elixir}
]}
]}
]},
{:@, [context: Elixir, import: Kernel],
[{:doc, [context: Elixir], ["Compute factorial"]}]},
{:def, [context: Elixir, import: Kernel],
[{:fac, [context: Elixir], [1]}, [do: 1]]},
{:def, [context: Elixir, import: Kernel],
[
{:fac, [context: Elixir], [{:n, [], Elixir}]},
[
do: {:*, [context: Elixir, import: Kernel],
[
{:n, [], Elixir},
{:fac, [],
[
{:-, [context: Elixir, import: Kernel],
[{:n, [], Elixir}, 1]}
]}
]}
]
]}
]}
]
]}
{:defmodule, [no_parens: true, line: 1],
[
{:__aliases__, [line: 1], [:Fac]},
[
{{:__block__, [format: :block, end_line: 8, line: 1], [:do]},
{:__block__, [],
[
{:@, [line: 2],
[
{:spec, [no_parens: true, line: 2],
[
{:::, [line: 2],
[
{:fac, [end_line: 2, line: 2],
[{:pos_integer, [no_parens: true, line: 2], nil}]},
{:pos_integer, [no_parens: true, line: 2], nil}
]}
]}
]},
{:@, [newlines: 1, line: 3],
[
{:doc, [no_parens: true, line: 3],
[{:__block__, [format: :string, line: 3], ["Compute factorial"]}]}
]},
{:def, [newlines: 1, no_parens: true, line: 4],
[
{:fac, [no_parens: true, line: 4],
[{:__block__, [original: '1', line: 4], [1]}]},
[
{{:__block__, [format: :block, end_line: 4, line: 4], [:do]},
{:__block__, [original: '1', line: 4], [1]}}
]
]},
{:def, [newlines: 1, no_parens: true, line: 5],
[
{:fac, [no_parens: true, line: 5],
[{:n, [no_parens: true, line: 5], nil}]},
[
{{:__block__, [format: :block, end_line: 7, line: 5], [:do]},
{:*, [line: 6], ...
Normal Quoted Code Formatter Quoted

27. Deeper inside
Code.format_string!
Code.Formatter.to_algebra(raw)
:elixir.string_to_tokens/4
:elixir.tokens_to_quoted/3
Code.Formatter.block_to_algebra/2

28. Deeper inside
Code.format_string!
{:doc_group,
{:doc_cons, {:doc_string, "defmodule", 9},
{:doc_force,
{:doc_cons,
{:doc_cons,
{:doc_group,
{:doc_cons, {:doc_cons, " ", {:doc_nest, "Fac", :cursor, :break}}, " do"},
:self},
{:doc_nest,
{:doc_cons, :doc_nil,
{:doc_cons, :doc_line,
{:doc_group,
{:doc_force,
{:doc_cons,
{:doc_cons,
{:doc_cons,
{:doc_cons,
{:doc_group,
{:doc_cons, {:doc_string, "@spec", 5},
{:doc_group,
{:doc_cons,
{:doc_cons, " ",
{:doc_nest, {:doc_cons, {...}, ...}, :cursor, :break}},
:doc_nil}, :self}}, :self},
{:doc_cons, :doc_line,
{:doc_group,
{:doc_cons, {:doc_string, "@doc", 4},
{:doc_group, {:doc_cons, {:doc_cons, " ", {...}}, :doc_nil},
:self}}, :self}}},
{:doc_cons, :doc_line, {:doc_group, "# stop condition", :self}}},
{:doc_cons, :doc_line,
{:doc_group,
{:doc_cons,
{:doc_cons, {:doc_string, "def", 3},
{:doc_force,
{:doc_cons,
{:doc_cons, {:doc_group, {:doc_cons, ...}, :self},
{:doc_nest, {...}, ...}}, {:doc_cons, :doc_line, "end"}}}},
{:doc_cons, {:doc_collapse, 2}, {:doc_break, "", :strict}}},
:self}}},
{:doc_cons, :doc_line,
{:doc_group,
{:doc_cons, {:doc_break, "", :strict},
{:doc_cons, {:doc_string, "def", 3},
{:doc_force,
{:doc_cons,...

29. Inspect.Algebra.format/2

31. Deeper inside
Code.format_string!
defmodule Fac do
@spec fac(pos_integer) :: pos_integer
@doc "Compute factorial"
# stop condition
def fac(1) do
1
end
def fac(n) do
n * fac(n - 1)
end
end
defmodule Fac do
@spec fac(pos_integer) :: pos_integer
@doc "Compute factorial"
def fac 1 do; 1 ;end # stop condition
def fac n do
n * fac n - 1
end
end

32. Deeper inside
Code.format_string!
Code.Formatter.to_algebra(raw)
:elixir.string_to_tokens/4
:elixir.tokens_to_quoted/3
Code.Formatter.block_to_algebra/2
Inspect.Algebra.format/2
# insert comments to the quoted code here

33. Deeper inside
Code.format_string!
%% elixir.erl
'string_to_quoted!'(String, StartLine, File, Opts) ->
case string_to_tokens(String, StartLine, File, Opts) of
{ok, Tokens} ->
case tokens_to_quoted(Tokens, File, Opts) of
{ok, Forms} ->
Forms;
{error, {Line, Error, Token}} ->
elixir_errors:parse_error(Line, File, Error, Token)
end;
{error, {Line, Error, Token}} ->
elixir_errors:parse_error(Line, File, Error, Token)
end.

35. $ echo 'IO.puts "Hello World" | mix format -
IO.puts("Hello World”)
$ mix format --check-formatted
** (Mix) mix format failed due to --check-formatted.
The following files were not formatted:
* lib/fac.ex
Useful tools wit the
formatter

36. Thanks