1.
@asgrim
Climbing the
Abstract Syntax Tree
James Titcumb
Forum PHP 2017

2.
$ whoami
James Titcumb
www.jamestitcumb.com
www.roave.com
@asgrim

3.
@asgrim
How PHP works
PHP code
OpCache
Execute (VM)
Lexer + Parser
Compiler

4.
@asgrim
The PHP Lexer
zend_language_scanner.l

5.
@asgrim
zend_language_scanner.l
<ST_IN_SCRIPTING>"exit" {
RETURN_TOKEN(T_EXIT);
}
<ST_IN_SCRIPTING>"die" {
RETURN_TOKEN(T_EXIT);
}
<ST_IN_SCRIPTING>"function" {
RETURN_TOKEN(T_FUNCTION);
}

6.
@asgrim
zend_language_scanner.l
<ST_IN_SCRIPTING>"exit" {
RETURN_TOKEN(T_EXIT);
}
<ST_IN_SCRIPTING>"die" {
RETURN_TOKEN(T_EXIT);
}
<ST_IN_SCRIPTING>"function" {
RETURN_TOKEN(T_FUNCTION);
}

7.
@asgrim
zend_language_scanner.l
<ST_IN_SCRIPTING>"exit" {
RETURN_TOKEN(T_EXIT);
}
<ST_IN_SCRIPTING>"die" {
RETURN_TOKEN(T_EXIT);
}
<ST_IN_SCRIPTING>"function" {
RETURN_TOKEN(T_FUNCTION);
}

8.
@asgrim
zend_language_scanner.l
<ST_IN_SCRIPTING>"exit" {
RETURN_TOKEN(T_EXIT);
}
<ST_IN_SCRIPTING>"die" {
RETURN_TOKEN(T_EXIT);
}
<ST_IN_SCRIPTING>"function" {
RETURN_TOKEN(T_FUNCTION);
}

9.
@asgrim
zend_language_scanner.l
<ST_DOUBLE_QUOTES,ST_BACKQUOTE,ST_HEREDOC>"${" {
yy_push_state(ST_LOOKING_FOR_VARNAME);
RETURN_TOKEN(T_DOLLAR_OPEN_CURLY_BRACES);
}
<ST_LOOKING_FOR_VARNAME>{LABEL}[[}] {
yyless(yyleng - 1);
zend_copy_value(zendlval, yytext, yyleng);
yy_pop_state();
yy_push_state(ST_IN_SCRIPTING);
RETURN_TOKEN(T_STRING_VARNAME);
}

10.
@asgrim
zend_language_scanner.l
<ST_DOUBLE_QUOTES,ST_BACKQUOTE,ST_HEREDOC>"${" {
yy_push_state(ST_LOOKING_FOR_VARNAME);
RETURN_TOKEN(T_DOLLAR_OPEN_CURLY_BRACES);
}
<ST_LOOKING_FOR_VARNAME>{LABEL}[[}] {
yyless(yyleng - 1);
zend_copy_value(zendlval, yytext, yyleng);
yy_pop_state();
yy_push_state(ST_IN_SCRIPTING);
RETURN_TOKEN(T_STRING_VARNAME);
}

11.
@asgrim
zend_language_scanner.l
<ST_DOUBLE_QUOTES,ST_BACKQUOTE,ST_HEREDOC>"${" {
yy_push_state(ST_LOOKING_FOR_VARNAME);
RETURN_TOKEN(T_DOLLAR_OPEN_CURLY_BRACES);
}
<ST_LOOKING_FOR_VARNAME>{LABEL}[[}] {
yyless(yyleng - 1);
zend_copy_value(zendlval, yytext, yyleng);
yy_pop_state();
yy_push_state(ST_IN_SCRIPTING);
RETURN_TOKEN(T_STRING_VARNAME);
}

12.
@asgrim
zend_language_scanner.l
<ST_DOUBLE_QUOTES,ST_BACKQUOTE,ST_HEREDOC>"${" {
yy_push_state(ST_LOOKING_FOR_VARNAME);
RETURN_TOKEN(T_DOLLAR_OPEN_CURLY_BRACES);
}
<ST_LOOKING_FOR_VARNAME>{LABEL}[[}] {
yyless(yyleng - 1);
zend_copy_value(zendlval, yytext, yyleng);
yy_pop_state();
yy_push_state(ST_IN_SCRIPTING);
RETURN_TOKEN(T_STRING_VARNAME);
}

13.
@asgrim
zend_language_scanner.l
<ST_DOUBLE_QUOTES,ST_BACKQUOTE,ST_HEREDOC>"${" {
yy_push_state(ST_LOOKING_FOR_VARNAME);
RETURN_TOKEN(T_DOLLAR_OPEN_CURLY_BRACES);
}
<ST_LOOKING_FOR_VARNAME>{LABEL}[[}] {
yyless(yyleng - 1);
zend_copy_value(zendlval, yytext, yyleng);
yy_pop_state();
yy_push_state(ST_IN_SCRIPTING);
RETURN_TOKEN(T_STRING_VARNAME);
}

14.
@asgrim
zend_language_scanner.l
<ST_DOUBLE_QUOTES,ST_BACKQUOTE,ST_HEREDOC>"${" {
yy_push_state(ST_LOOKING_FOR_VARNAME);
RETURN_TOKEN(T_DOLLAR_OPEN_CURLY_BRACES);
}
<ST_LOOKING_FOR_VARNAME>{LABEL}[[}] {
yyless(yyleng - 1);
zend_copy_value(zendlval, yytext, yyleng);
yy_pop_state();
yy_push_state(ST_IN_SCRIPTING);
RETURN_TOKEN(T_STRING_VARNAME);
}

15.
@asgrim
zend_language_scanner.l
<ST_DOUBLE_QUOTES,ST_BACKQUOTE,ST_HEREDOC>"${" {
yy_push_state(ST_LOOKING_FOR_VARNAME);
RETURN_TOKEN(T_DOLLAR_OPEN_CURLY_BRACES);
}
<ST_LOOKING_FOR_VARNAME>{LABEL}[[}] {
yyless(yyleng - 1);
zend_copy_value(zendlval, yytext, yyleng);
yy_pop_state();
yy_push_state(ST_IN_SCRIPTING);
RETURN_TOKEN(T_STRING_VARNAME);
}

16.
@asgrim
zend_language_scanner.l
<ST_DOUBLE_QUOTES,ST_BACKQUOTE,ST_HEREDOC>"${" {
yy_push_state(ST_LOOKING_FOR_VARNAME);
RETURN_TOKEN(T_DOLLAR_OPEN_CURLY_BRACES);
}
<ST_LOOKING_FOR_VARNAME>{LABEL}[[}] {
yyless(yyleng - 1);
zend_copy_value(zendlval, yytext, yyleng);
yy_pop_state();
yy_push_state(ST_IN_SCRIPTING);
RETURN_TOKEN(T_STRING_VARNAME);
}

17.
@asgrim
zend_language_scanner.l
<ST_DOUBLE_QUOTES,ST_BACKQUOTE,ST_HEREDOC>"${" {
yy_push_state(ST_LOOKING_FOR_VARNAME);
RETURN_TOKEN(T_DOLLAR_OPEN_CURLY_BRACES);
}
<ST_LOOKING_FOR_VARNAME>{LABEL}[[}] {
yyless(yyleng - 1);
zend_copy_value(zendlval, yytext, yyleng);
yy_pop_state();
yy_push_state(ST_IN_SCRIPTING);
RETURN_TOKEN(T_STRING_VARNAME);
}

18.
@asgrim
The PHP Lexer
zend_language_scanner.l

19.
@asgrim
The PHP Lexer
zend_language_scanner.l
re2c

20.
@asgrim
The PHP Lexer
zend_language_scanner.l
re2c
zend_language_scanner.c

21.
@asgrim
The PHP Parser
zend_language_parser.y

22.
@asgrim
zend_language_parser.y
if_stmt:
if_stmt_without_else %prec T_NOELSE { $$ = $1; }
| if_stmt_without_else T_ELSE statement
{ $$ = zend_ast_list_add($1, zend_ast_create(ZEND_AST_IF_ELEM, NULL, $3)); }
;
if_stmt_without_else:
T_IF '(' expr ')' statement
{ $$ = zend_ast_create_list(1, ZEND_AST_IF,
zend_ast_create(ZEND_AST_IF_ELEM, $3, $5)); }
| if_stmt_without_else T_ELSEIF '(' expr ')' statement
{ $$ = zend_ast_list_add($1,
zend_ast_create(ZEND_AST_IF_ELEM, $4, $6)); }
;

23.
@asgrim
if_stmt:
if_stmt_without_else %prec T_NOELSE { $$ = $1; }
| if_stmt_without_else T_ELSE statement
{ $$ = zend_ast_list_add($1, zend_ast_create(ZEND_AST_IF_ELEM, NULL, $3)); }
;
if_stmt_without_else:
T_IF '(' expr ')' statement
{ $$ = zend_ast_create_list(1, ZEND_AST_IF,
zend_ast_create(ZEND_AST_IF_ELEM, $3, $5)); }
| if_stmt_without_else T_ELSEIF '(' expr ')' statement
{ $$ = zend_ast_list_add($1,
zend_ast_create(ZEND_AST_IF_ELEM, $4, $6)); }
;
zend_language_parser.y

24.
@asgrim
if_stmt:
if_stmt_without_else %prec T_NOELSE { $$ = $1; }
| if_stmt_without_else T_ELSE statement
{ $$ = zend_ast_list_add($1, zend_ast_create(ZEND_AST_IF_ELEM, NULL, $3)); }
;
if_stmt_without_else:
T_IF '(' expr ')' statement
{ $$ = zend_ast_create_list(1, ZEND_AST_IF,
zend_ast_create(ZEND_AST_IF_ELEM, $3, $5)); }
| if_stmt_without_else T_ELSEIF '(' expr ')' statement
{ $$ = zend_ast_list_add($1,
zend_ast_create(ZEND_AST_IF_ELEM, $4, $6)); }
;
zend_language_parser.y

25.
@asgrim
if_stmt:
if_stmt_without_else %prec T_NOELSE { $$ = $1; }
| if_stmt_without_else T_ELSE statement
{ $$ = zend_ast_list_add($1, zend_ast_create(ZEND_AST_IF_ELEM, NULL, $3)); }
;
if_stmt_without_else:
T_IF '(' expr ')' statement
{ $$ = zend_ast_create_list(1, ZEND_AST_IF,
zend_ast_create(ZEND_AST_IF_ELEM, $3, $5)); }
| if_stmt_without_else T_ELSEIF '(' expr ')' statement
{ $$ = zend_ast_list_add($1,
zend_ast_create(ZEND_AST_IF_ELEM, $4, $6)); }
;
zend_language_parser.y

26.
@asgrim
if_stmt:
if_stmt_without_else %prec T_NOELSE { $$ = $1; }
| if_stmt_without_else T_ELSE statement
{ $$ = zend_ast_list_add($1, zend_ast_create(ZEND_AST_IF_ELEM, NULL, $3)); }
;
if_stmt_without_else:
T_IF '(' expr ')' statement
{ $$ = zend_ast_create_list(1, ZEND_AST_IF,
zend_ast_create(ZEND_AST_IF_ELEM, $3, $5)); }
| if_stmt_without_else T_ELSEIF '(' expr ')' statement
{ $$ = zend_ast_list_add($1,
zend_ast_create(ZEND_AST_IF_ELEM, $4, $6)); }
;
zend_language_parser.y

27.
@asgrim
if_stmt:
if_stmt_without_else %prec T_NOELSE { $$ = $1; }
| if_stmt_without_else T_ELSE statement
{ $$ = zend_ast_list_add($1, zend_ast_create(ZEND_AST_IF_ELEM, NULL, $3)); }
;
if_stmt_without_else:
T_IF '(' expr ')' statement
{ $$ = zend_ast_create_list(1, ZEND_AST_IF,
zend_ast_create(ZEND_AST_IF_ELEM, $3, $5)); }
| if_stmt_without_else T_ELSEIF '(' expr ')' statement
{ $$ = zend_ast_list_add($1,
zend_ast_create(ZEND_AST_IF_ELEM, $4, $6)); }
;
zend_language_parser.y

28.
@asgrim
if_stmt:
if_stmt_without_else %prec T_NOELSE { $$ = $1; }
| if_stmt_without_else T_ELSE statement
{ $$ = zend_ast_list_add($1, zend_ast_create(ZEND_AST_IF_ELEM, NULL, $3)); }
;
if_stmt_without_else:
T_IF '(' expr ')' statement
{ $$ = zend_ast_create_list(1, ZEND_AST_IF,
zend_ast_create(ZEND_AST_IF_ELEM, $3, $5)); }
| if_stmt_without_else T_ELSEIF '(' expr ')' statement
{ $$ = zend_ast_list_add($1,
zend_ast_create(ZEND_AST_IF_ELEM, $4, $6)); }
;
zend_language_parser.y

29.
@asgrim
if_stmt:
if_stmt_without_else %prec T_NOELSE { $$ = $1; }
| if_stmt_without_else T_ELSE statement
{ $$ = zend_ast_list_add($1, zend_ast_create(ZEND_AST_IF_ELEM, NULL, $3)); }
;
if_stmt_without_else:
T_IF '(' expr ')' statement
{ $$ = zend_ast_create_list(1, ZEND_AST_IF,
zend_ast_create(ZEND_AST_IF_ELEM, $3, $5)); }
| if_stmt_without_else T_ELSEIF '(' expr ')' statement
{ $$ = zend_ast_list_add($1,
zend_ast_create(ZEND_AST_IF_ELEM, $4, $6)); }
;
zend_language_parser.y

30.
@asgrim
if ($a == 1)
{
a();
}
else if ($b == 1)
{
b();
}
else
{
c();
}
Using the rules to parse

31.
@asgrim
if ($a == 1)
{
a();
}
else if ($b == 1)
{
b();
}
else
{
c();
}
Using the rules to parse
if_stmt_without_else (A)

32.
@asgrim
if ($a == 1)
{
a();
}
else if ($b == 1)
{
b();
}
else
{
c();
}
Using the rules to parse
if_stmt_without_else (A)
if_stmt_without_else (B)

33.
@asgrim
if ($a == 1)
{
a();
}
else if ($b == 1)
{
b();
}
else
{
c();
}
Using the rules to parse
if_stmt_without_else (A)
if_stmt_without_else (B)
if_stmt

34.
@asgrim
Zend_language_parser.y (PHP 7.0.10)
if_stmt:
if_stmt_without_else %prec T_NOELSE { $$ = $1; }
| if_stmt_without_else T_ELSE statement
{ $$ = zend_ast_list_add($1, zend_ast_create(ZEND_AST_IF_ELEM, NULL, $3)); }
;
if_stmt_without_else:
T_IF '(' expr ')' statement
{ $$ = zend_ast_create_list(1, ZEND_AST_IF,
zend_ast_create(ZEND_AST_IF_ELEM, $3, $5)); }
| if_stmt_without_else T_ELSEIF '(' expr ')' statement
{ $$ = zend_ast_list_add($1,
zend_ast_create(ZEND_AST_IF_ELEM, $4, $6)); }
;

35.
@asgrim
zend_language_parser.y (PHP 5.6.26)
T_IF parenthesis_expr { zend_do_if_cond(&$2, &$1 TSRMLS_CC); }
statement { zend_do_if_after_statement(&$1, 1 TSRMLS_CC); }
void zend_do_if_cond(const znode *cond, znode *closing_bracket_token TSRMLS_DC)
{
int if_cond_op_number = get_next_op_number(CG(active_op_array));
zend_op *opline = get_next_op(CG(active_op_array) TSRMLS_CC);
opline->opcode = ZEND_JMPZ;
SET_NODE(opline->op1, cond);
closing_bracket_token->u.op.opline_num = if_cond_op_number;
SET_UNUSED(opline->op2);
INC_BPC(CG(active_op_array));
}

36.
@asgrim
AST is new in PHP 7+

37.
@asgrim
How PHP works
PHP code
OpCache
Execute (VM)
Lexer + Parser
Compiler

38.
@asgrim
Let’s simplify!

39.
@asgrim
First… WTF is AST?

40.
@asgrim
AST is just a data structure

41.
@asgrim
PHP code
<?php
echo "Hello world";

42.
@asgrim
An AST representation
Echo statement
`-- String, value "Hello world"

43.
@asgrim
PHP code
<?php
echo "Hello " . "world";

44.
@asgrim
An AST representation
Echo statement
`-- Concat
|-- Left
| `-- String, value "Hello "
`-- Right
`-- String, value "world"

45.
@asgrim
PHP code
<?php
$a = 5;
$b = 3;
echo $a + ($b * 2);

46.
@asgrim
An AST representation
Assign statement
|-- Variable $a
`-- Integer, value 5
Assign statement
|-- Variable $b
`-- Integer, value 3
Echo statement
`-- Add operation
|-- Left
| `-- Variable $a
`-- Right
`-- Multiply operation
|-- Left
| `-- Variable $b
`-- Right
`-- Integer, value 2

47.
@asgrim
Why?

48.
@asgrim
Faster!*

49.
@asgrim
AST compilation
Statements
EchoAssign
Scalar
value: (int)5
Variable
name: $a
Assign
Scalar
value: (int)3
Variable
name: $b
Add op
Right operandLeft operand
Variable
name: $a
Multiply op
Right operandLeft operand
Variable
name: $b
Scalar
value: (int)2

50.
@asgrim
AST compilation: pre-order traversal
Statements
EchoAssign
Scalar
value: (int)5
Variable
name: $a
Assign
Scalar
value: (int)3
Variable
name: $b
Add op
Right operandLeft operand
Variable
name: $a
Multiply op
Right operandLeft operand
Variable
name: $b
Scalar
value: (int)2

51.
@asgrim
Pre-order traversal: Polish notation
Assign(Variable $a, Scalar 5)
Assign(Variable $b, Scalar 3)
Echo (
Add(
Variable $a,
Multiply( $b, 2 )
)
)

52.
@asgrim
Order of precedence
1 + 2 * 3
= 1 + (2 * 3) = 7?
= (1 + 2) * 3 = 9?

53.
@asgrim
Order of precedence
1 + 2 * 3
= 1 + (2 * 3) = 7?
= (1 + 2) * 3 = 9?
+ 1 * 2 3

54.
@asgrim
Order of precedence
1 + 2 * 3
= 1 + (2 * 3) = 7?
= (1 + 2) * 3 = 9?
+ 1 * 2 3
Operator Left operand
Right operand

55.
@asgrim
Order of precedence
1 + 2 * 3
= 1 + (2 * 3) = 7?
= (1 + 2) * 3 = 9?
+ 1 * 2 3
Operator Left operand Right operand
Operator Left operand
Right operand

56.
@asgrim
Reverse Polish Notation
1 2 3 * +

57.
@asgrim
Reverse Polish Notation
1 2 3 * + The stack

58.
@asgrim
Reverse Polish Notation
1 2 3 * + The stack
1

59.
@asgrim
Reverse Polish Notation
1 2 3 * + The stack
1
2

60.
@asgrim
Reverse Polish Notation
1 2 3 * + The stack
1
2
3

61.
@asgrim
Reverse Polish Notation
1 2 3 * + The stack
1
2
3

62.
@asgrim
Reverse Polish Notation
1 2 3 * + The stack
1
2
3

63.
@asgrim
Reverse Polish Notation
1 2 3 * + The stack
1
6

64.
@asgrim
Reverse Polish Notation
1 2 3 * + The stack
1
6

65.
@asgrim
Reverse Polish Notation
1 2 3 * + The stack
7

66.
@asgrim
Let’s write a compiler (!!!)
In three easy steps…

67.
@asgrim
Warning: do not use in production

68.
@asgrim
View > Source
https://github.com/asgrim/basic-maths-compiler

69.
@asgrim
Define the language
Tokens
● T_ADD (+)
● T_SUBTRACT (-)
● T_MULTIPLY (/)
● T_DIVIDE (*)
● T_INTEGER (d)
● T_WHITESPACE (s+)

70.
@asgrim
Step 1: Writing a simple lexer

71.
@asgrim
Using regular expressions
private static $matches = [
'/^(+)/' => Token::T_ADD,
'/^(-)/' => Token::T_SUBTRACT,
'/^(*)/' => Token::T_MULTIPLY,
'/^(/)/' => Token::T_DIVIDE,
'/^(d+)/' => Token::T_INTEGER,
'/^(s+)/' => Token::T_WHITESPACE,
];

72.
@asgrim
Step through the input string
public function __invoke(string $input) : array
{
$tokens = [];
$offset = 0;
while ($offset < strlen($input)) {
$focus = substr($input, $offset);
$result = $this->match($focus);
$tokens[] = $result;
$offset += strlen($result->getLexeme());
}
return $tokens;
}

73.
@asgrim
The matching method
private function match(string $input) : Token
{
foreach (self::$matches as $pattern => $token) {
if (preg_match($pattern, $input, $matches)) {
return new Token($token, $matches[1]);
}
}
throw new RuntimeException(sprintf(
'Unmatched token, next 15 chars were: %s', substr($input, 0, 15)
));
}

74.
@asgrim
Step 2: Parsing the tokens

75.
@asgrim
Order tokens by operator precedence
/**
* Higher number is higher precedence.
* @var int[]
*/
private static $operatorPrecedence = [
Token::T_SUBTRACT => 0,
Token::T_ADD => 1,
Token::T_DIVIDE => 2,
Token::T_MULTIPLY => 3,
];

76.
@asgrim
Order tokens by operator precedence
/** @var Token[] $stack */
$stack = [];
/** @var Token[] $operators */
$operators = [];
while (false !== ($token = current($tokens))) {
if ($token->isOperator()) {
// ...
}
$stack[] = $token;
next($tokens);
}

77.
@asgrim
Order tokens by operator precedence
/** @var Token[] $stack */
$stack = [];
/** @var Token[] $operators */
$operators = [];
while (false !== ($token = current($tokens))) {
if ($token->isOperator()) {
// ...
}
$stack[] = $token;
next($tokens);
}

78.
@asgrim
Order tokens by operator precedence
/** @var Token[] $stack */
$stack = [];
/** @var Token[] $operators */
$operators = [];
while (false !== ($token = current($tokens))) {
if ($token->isOperator()) {
// ...
}
$stack[] = $token;
next($tokens);
}

79.
@asgrim
Order tokens by operator precedence
/** @var Token[] $stack */
$stack = [];
/** @var Token[] $operators */
$operators = [];
while (false !== ($token = current($tokens))) {
if ($token->isOperator()) {
// ...
}
$stack[] = $token;
next($tokens);
}

80.
@asgrim
Order tokens by operator precedence
if ($token->isOperator()) {
$tokenPrecedence = self::$operatorPrecedence[$token->getToken()];
while (
count($operators)
&& self::$operatorPrecedence[$operators[count($operators) - 1]->getToken()]
> $tokenPrecedence
) {
$higherOp = array_pop($operators);
$stack[] = $higherOp;
}
$operators[] = $token;
next($tokens);
continue;
}

81.
@asgrim
Order tokens by operator precedence
if ($token->isOperator()) {
$tokenPrecedence = self::$operatorPrecedence[$token->getToken()];
while (
count($operators)
&& self::$operatorPrecedence[$operators[count($operators) - 1]->getToken()]
> $tokenPrecedence
) {
$higherOp = array_pop($operators);
$stack[] = $higherOp;
}
$operators[] = $token;
next($tokens);
continue;
}

82.
@asgrim
Order tokens by operator precedence
if ($token->isOperator()) {
$tokenPrecedence = self::$operatorPrecedence[$token->getToken()];
while (
count($operators)
&& self::$operatorPrecedence[$operators[count($operators) - 1]->getToken()]
> $tokenPrecedence
) {
$higherOp = array_pop($operators);
$stack[] = $higherOp;
}
$operators[] = $token;
next($tokens);
continue;
}

83.
@asgrim
Order tokens by operator precedence
if ($token->isOperator()) {
$tokenPrecedence = self::$operatorPrecedence[$token->getToken()];
while (
count($operators)
&& self::$operatorPrecedence[$operators[count($operators) - 1]->getToken()]
> $tokenPrecedence
) {
$higherOp = array_pop($operators);
$stack[] = $higherOp;
}
$operators[] = $token;
next($tokens);
continue;
}

84.
@asgrim
Order tokens by operator precedence
// Clean up by moving any remaining operators onto the token stack
while (count($operators)) {
$stack[] = array_pop($operators);
}
return $stack;

85.
@asgrim
Order tokens by operator precedence
1 + 2 * 3
Output stack
Operator stack

86.
@asgrim
Order tokens by operator precedence
1 + 2 * 3
1Output stack
Operator stack

87.
@asgrim
Order tokens by operator precedence
1 + 2 * 3
1
+
Output stack
Operator stack

88.
@asgrim
Order tokens by operator precedence
1 + 2 * 3
1 2
+
Output stack
Operator stack

89.
@asgrim
Order tokens by operator precedence
1 + 2 * 3
1 2
+ *
Output stack
Operator stack

90.
@asgrim
Order tokens by operator precedence
1 + 2 * 3
1 2 3
+ *
Output stack
Operator stack

91.
@asgrim
Order tokens by operator precedence
1 + 2 * 3
1 2 3 *
+ *
Output stack
Operator stack

92.
@asgrim
Order tokens by operator precedence
1 + 2 * 3
1 2 3 * +
+
Output stack
Operator stack

93.
@asgrim
Create AST
while ($ip < count($tokenStack)) {
$token = $tokenStack[$ip++];
if ($token->isOperator()) {
// (figure out $nodeType)
$right = array_pop($astStack);
$left = array_pop($astStack);
$astStack[] = new $nodeType($left, $right);
continue;
}
$astStack[] = new NodeScalarIntegerValue((int)$token->getLexeme());
}

94.
@asgrim
Create AST
while ($ip < count($tokenStack)) {
$token = $tokenStack[$ip++];
if ($token->isOperator()) {
// (figure out $nodeType)
$right = array_pop($astStack);
$left = array_pop($astStack);
$astStack[] = new $nodeType($left, $right);
continue;
}
$astStack[] = new NodeScalarIntegerValue((int)$token->getLexeme());
}

95.
@asgrim
Create AST
while ($ip < count($tokenStack)) {
$token = $tokenStack[$ip++];
if ($token->isOperator()) {
// (figure out $nodeType)
$right = array_pop($astStack);
$left = array_pop($astStack);
$astStack[] = new $nodeType($left, $right);
continue;
}
$astStack[] = new NodeScalarIntegerValue((int)$token->getLexeme());
}

96.
@asgrim
Create AST
while ($ip < count($tokenStack)) {
$token = $tokenStack[$ip++];
if ($token->isOperator()) {
// (figure out $nodeType)
$right = array_pop($astStack);
$left = array_pop($astStack);
$astStack[] = new $nodeType($left, $right);
continue;
}
$astStack[] = new NodeScalarIntegerValue((int)$token->getLexeme());
}

97.
@asgrim
Create AST
NodeBinaryOpAdd (
NodeScalarIntegerValue(1),
NodeBinaryOpMultiply (
NodeScalarIntegerValue(2),
NodeScalarIntegerValue(3)
)
)

98.
@asgrim
Step 3: Executing the AST

99.
@asgrim
Compile & execute AST
private function compileNode(NodeInterface $node)
{
if ($node instanceof NodeBinaryOpAbstractBinaryOp) {
return $this->compileBinaryOp($node);
}
if ($node instanceof NodeScalarIntegerValue) {
return $node->getValue();
}
}

100.
@asgrim
Compile & execute AST
private function compileBinaryOp(NodeBinaryOpAbstractBinaryOp $node)
{
$left = $this->compileNode($node->getLeft());
$right = $this->compileNode($node->getRight());
switch (get_class($node)) {
case NodeBinaryOpAdd::class:
return $left + $right;
case NodeBinaryOpSubtract::class:
return $left - $right;
case NodeBinaryOpMultiply::class:
return $left * $right;
case NodeBinaryOpDivide::class:
return $left / $right;
}
}

101.
@asgrim
What does this mean for me?

102.
@asgrim
AST in userland

103.
@asgrim
php-ast extension
https://github.com/nikic/php-ast

104.
@asgrim
php-ast example usage
<?php
require 'path/to/util.php';
$code = <<<'EOC'
<?php
$var = 42;
EOC;
echo ast_dump(astparse_code($code, $version=35)), "n";
// Output:
AST_STMT_LIST
0: AST_ASSIGN
var: AST_VAR
name: "var"
expr: 42

105.
@asgrim
astkit
https://github.com/sgolemon/astkit

106.
@asgrim
astkit example usage
$if = AstKit::parseString(<<<EOD
if (true) {
echo "This is a triumph.n";
} else {
echo "The cake is a lie.n";
}
EOD
);
$if->execute(); // First run, program is as-seen above
$const = $if->getChild(0)->getChild(0);
// Replace the "true" constant in the condition with false
$const->graft(0, false);
// Can also graft other AstKit nodes, instead of constants
$if->execute(); // Second run now takes the else path

107.
@asgrim
PhpParser
https://github.com/nikic/PHP-Parser

108.
@asgrim
PHP Parser
<?php
use PhpParserParserFactory;
$parser = (new ParserFactory)
->create(ParserFactory::PREFER_PHP7);
print_r($parser->parse(
file_get_contents('ast-demo-src.php')
));

109.
@asgrim
Better Reflection
https://github.com/Roave/BetterReflection

110.
@asgrim
Better Reflection workflow
Reflector
Source
Locator
PhpParser
Reflection

111.
@asgrim
PHP Reflection
$reflection = new ReflectionClass(
MyExampleClass::class
);
$this->assertSame(
'ExampleClass',
$reflection->getShortName()
);

112.
@asgrim
Better Reflection
$reflection = (new BetterReflection())
->classReflector()
->reflect(MyExampleClass::class);
$this->assertSame(
'ExampleClass',
$reflection->getShortName()
);

113.
@asgrim
Class BetterReflection
public function sourceLocator() : SourceLocator
{
$astLocator = $this->astLocator();
return $this->sourceLocator
?? $this->sourceLocator = new MemoizingSourceLocator(new AggregateSourceLocator([
new PhpInternalSourceLocator($astLocator),
new EvaledCodeSourceLocator($astLocator),
new AutoloadSourceLocator($astLocator),
]));
}
public function classReflector() : ClassReflector
{
return $this->classReflector
?? $this->classReflector = new ClassReflector($this->sourceLocator());
}

114.
@asgrim
Given a class structure...
<?php
class Foo
{
private $bar;
public function thing()
{
}
}

115.
@asgrim
… we get the AST!
Class, name Foo
|-- Statements
| |-- Property, name bar
| | |-- Type [private]
| | `-- Attributes [start line: 7, end line: 9]
| `-- Method, name thing
| |-- Type [public]
| |-- Parameters [...]
| |-- Statements [...]
| `-- Attributes [start line: 7, end line: 9]
`-- Attributes [start line: 3, end line: 10]

116.
@asgrim
What can I use Better Reflection for?

117.
@asgrim
Monkey patching example
class MyClass
{
public function foo()
{
return 5;
}
}

118.
@asgrim
Monkey patching example
use RoaveBetterReflectionReflectorClassReflector;
use RoaveBetterReflectionSourceLocatorTypeSingleFileSourceLocator;
use RoaveBetterReflectionUtilAutoloadClassLoader;
use RoaveBetterReflectionUtilAutoloadClassLoaderMethodFileCacheLoader;
$loader = new ClassLoader(FileCacheLoader::defaultFileCacheLoader(__DIR__));
// Create the reflection first (without loading)
$classInfo = (new ClassReflector(
new SingleFileSourceLocator(
__DIR__ . '/MyClass.php',
(new BetterReflection())->astLocator()
)
))->reflect('MyClass');
$loader->addClass($classInfo);

119.
@asgrim
Monkey patching example
use RoaveBetterReflectionReflectorClassReflector;
use RoaveBetterReflectionSourceLocatorTypeSingleFileSourceLocator;
use RoaveBetterReflectionUtilAutoloadClassLoader;
use RoaveBetterReflectionUtilAutoloadClassLoaderMethodFileCacheLoader;
$loader = new ClassLoader(FileCacheLoader::defaultFileCacheLoader(__DIR__));
// Create the reflection first (without loading)
$classInfo = (new ClassReflector(
new SingleFileSourceLocator(
__DIR__ . '/MyClass.php',
(new BetterReflection())->astLocator()
)
))->reflect('MyClass');
$loader->addClass($classInfo);

120.
@asgrim
Monkey patching example
// Override the body...!
$classInfo->getMethod('foo')->setBodyFromClosure(
function () {
return 4;
}
);
$c = new MyClass();
echo $c->foo() . "n";

121.
@asgrim
Monkey patching example
// Override the body...!
$classInfo->getMethod('foo')->setBodyFromClosure(
function () {
return 4;
}
);
$c = new MyClass();
echo $c->foo() . "n"; // returns 4, not 5

122.
@asgrim
To summarise
● For PHP engine:
○ AST is an efficient data structure to represent code
○ AST means faster compilation (ignoring opcache)
○ Separation in PHP engine for parser and compiler
○ https://wiki.php.net/rfc/abstract_syntax_tree
● Concepts can be used in userland
○ PHP Parser library - https://github.com/nikic/php-parser
○ Better Reflection - https://github.com/Roave/BetterReflection
■ Reflect on not-yet-loaded files
■ Monkey patching in userland code (!)
○ Static analysis opportunities
■ Better Reflection
■ Exakat static analysis (uses own AST)
■ Phan (uses php-ast extension)

123.
Any questions?
Please leave feedback!
https://joind.in/talk/6dbd4
James Titcumb
@asgrim