The pug templating engine is designed to be easily extensible, as well as having powerful core functionality. It was recently re-written to be more modular and structured. This talk discusses how a compiler is built out of a series of independent stages.

2. HTML Output
<article>
<h1>Title</h1>
<p>Body</p>
</article>

3. Pug Source
article
h1 Title
p Body

4. Pug Compiler
Lexer Parser Code Gen

5. Lexer
article
h1 Title
p Body
• tag – "article"
• indent
• tag – "h1"
• text – "Title"
• new line
• tag – "p"
• text – "Body"
• outdent

6. Lexer
class Lexer {
constructor(str) {
this.str = str;
this.indent = 0;
}
}

7. Lexer
tag() {
let match = /^[a-z]+/.exec(this.str);
if (match) {
this.str = this.str.substr(match[0].length);
return {type: 'tag', val: match[0]};
}
}

8. Lexer
text() {
let match = /^ .+/.exec(this.str);
if (match) {
this.str = this.str.substr(match[0].length);
return {type: 'text', val: match[0].substr(1)};
}
}

9. Lexer
newline() {
let match = /^n( *)/.exec(this.str);
if (match) {
this.str = this.str.substr(match[0].length);
let oldIndent = this.indent;
this.indent = match[1].length;
if (this.indent > oldIndent) return {type: 'indent'};
if (this.indent < oldIndent) return {type: 'outdent'};
return {type: 'newline'};
}
}

10. Lexer
fail() {
throw new Error(
'Unexpected text'
);
}
getTokens() {
let tokens = [];
while (this.str.length) {
tokens.push(
this.newline() ||
this.tag() ||
this.text() ||
this.fail()
);
}
tokens.push({type: 'eos'});
return tokens;
}

11. Parser
OutdentIndent
New
Line
Text
"Body"
Tag
"h1"
Tag
"article"
Text
"Title"
Tag
"p"

12. Parser
Text
"Body"
Tag
"h1"
Tag
"article"
Text
"Title"
Tag
"p"

13. Parser
class Parser {
constructor(tokens) {
this.tokens = new TokenStream(tokens);
}
}

14. Token Stream
• Next – get the next token and advance the stream
• Peek – get the next token without advancing the stream
• Expect(type) – get the next token and advance the stream

15. Parser
parseFile() {
let nodes = [];
while (this.tokens.peek().type !== 'eos') {
if (this.tokens.peek().type === 'tag') {
nodes.push(this.parseTag());
} else {
this.tokens.expect('newline');
}
}
this.tokens.expect('eos');
return {type: 'File', nodes};
}

16. Parser
parseTag() {
let tok = this.tokens.expect('tag');
let body = null;
switch (this.tokens.peek().type) {
case 'text':
body = this.parseText();
break;
case 'indent':
body = this.parseBlock();
break;
}
return {type: 'Tag', name: tok.val, body};
}

17. Parser
parseBlock() {
this.tokens.expect('indent');
let nodes = [];
while (this.tokens.peek().type !== 'outdent') {
if (this.tokens.peek().type === 'tag') {
nodes.push(this.parseTag());
} else {
this.tokens.expect('newline');
}
}
this.tokens.expect('outdent');
return {type: 'Block', nodes};
}

18. Code Gen
Text
"Body"
Tag
"h1"
Tag
"article"
Text
"Title"
Tag
"p"

19. Code Gen
Text
"Body"
Tag
"h1"
Tag
"article"
"Title"
Tag
"p"

20. Code Gen
Text
"Body"
"<h1>Title</h1>"
Tag
"article"
Tag
"p"

21. Code Gen
"Body"
"<h1>Title</h1>"
Tag
"article"
Tag
"p"

22. Code Gen
"<h1>Title</h1>"
Tag
"article"
"<p>Body</p>"

23. Code Gen "<article><h1>Title</h1><p>Body</p></article>"

24. Code Gen
function render(node) {
switch (node.type) {
case 'Block':
return renderBlock(node);
case 'Tag':
return renderTag(node);
case 'Text':
return renderText(node);
}
}

25. Code Gen
function renderBlock(node) {
return node.nodes.map(render).join('n');
}

26. Code Gen
function renderTag(node) {
if (!node.body) return '<' + node.name + '/>';
return (
'<' + node.name + '>' +
render(node.body) +
'</' + node.name + '>'
);
}

27. Code Gen
function renderText(node) {
return node.val;
}

28. Pug Compiler
Lexer Parser Code Gen

29. Includes
article
h1 Title
include ./content.pug

30. Linking
File A
Include
File B
FILE B

31. Linking
File A
Include
File B
FILE B

32. Pug Compiler Pipeline
Lexer Parser Loader Linker Code-Gen
String Tokens AST Collection
of ASTs
AST String

33. Lexer Parser Loader Linker Code-Gen
Now it's your turn!
String Tokens AST Collection
of ASTs
AST String