SlideShare a Scribd company logo
1 of 66
Download to read offline
Осторожно, DSL!
Цыганов Иван
Positive Technologies
server {
location / {
proxy_pass http://localhost:8080/;
}
location ~ .(gif|jpg|png)$ {
root /data/images;
}
}
version: '2'
services:
db:
image: postgres
web:
build: .
command: python manage.py runserver 0.0.0.0:8000
volumes:
- .:/code
ports:
- "8000:8000"
depends_on:
- db
DOCKER COMPOSE
LOGROTATE
compress
"/var/log/httpd/access.log" /var/log/httpd/error.log {
rotate 5
size=100k
sharedscripts
nocompress
postrotate
/sbin/killall -HUP httpd
endscript
}
/var/log/messages {
rotate 5
weekly
postrotate
/sbin/killall -HUP syslogd
endscript
}
compress
"/var/log/httpd/access.log" /var/log/httpd/error.log {
rotate 5
size=100k
sharedscripts
nocompress
postrotate
/sbin/killall -HUP httpd
endscript
}
config = {

'compress': True,

('/var/log/nginx/access.log', '/var/log/nginx/error.log'): {

'rotate': 5,

'size': 102400,
'compress': False,

'sharedscripts': True,

'postrotate': [

'/sbin/killall -HUP syslogd'

]

}

}

compress
"/var/log/httpd/access.log" /var/log/httpd/error.log {
rotate 5
size=100k
sharedscripts
nocompress
postrotate
/sbin/killall -HUP httpd
endscript
}
config = {

'compress': True,

('/var/log/nginx/access.log', '/var/log/nginx/error.log'): {

'rotate': 5,

'size': 102400,
'compress': False,

'sharedscripts': True,

'postrotate': [

'/sbin/killall -HUP syslogd'

]

}

}

compress
"/var/log/httpd/access.log" /var/log/httpd/error.log {
rotate 5
size=100k
sharedscripts
nocompress
postrotate
/sbin/killall -HUP httpd
endscript
}
size=100k
'size': 102400,
config = {

'compress': True,

('/var/log/nginx/access.log', '/var/log/nginx/error.log'): {

'rotate': 5,

'size': 102400,
'postrotate': [

'/sbin/killall -HUP syslogd'

]

}

}

'size': '100 KB',
config = {

'compress': True,

('/var/log/nginx/access.log', '/var/log/nginx/error.log'): {

'rotate': 5,

'size': 102400,
'postrotate': [

'/sbin/killall -HUP syslogd'

]

}

}

'size': '100 KB',
def get_size(self, value):

size_value, _, size_unit = value.partition(' ')

return self.units_to_bytes(size_value, size_unit)
config = {

'compress': True,

('/var/log/nginx/access.log', '/var/log/nginx/error.log'): {

'rotate': 5,

'size': 102400,
'postrotate': [

'/sbin/killall -HUP syslogd'

]

}

}

'size': '1MB + 100KB',
config = {

'compress': True,

('/var/log/nginx/access.log', '/var/log/nginx/error.log'): {

'rotate': 5,

'size': 102400,
'postrotate': [

'/sbin/killall -HUP syslogd'

]

}

}

'size': '1MB + 100KB',
parse_size = re.compile(

pattern=r'''

(?P<Value>d+)s*

(?P<ValueUnit>KB|MB|GB)?s*

((?P<Operator>[-+/*])s*)?

(

(?P<Delta>d+)s*

(?P<DeltaUnit>KB|MB|GB)?

)?
''',

flags=re.IGNORECASE | re.VERBOSE

).search
config = {

'compress': True,

('/var/log/nginx/access.log', '/var/log/nginx/error.log'): {

'rotate': 5,

'size': 102400,
'postrotate': [

'/sbin/killall -HUP syslogd'

]

}

}

'size': ‘100MB - 2 * (1MB + 100KB)’,
Some people, when confronted with a problem,
think "I know, I’ll use regular expressions."
Now they have two problems.
— Jamie Zawinski
Осторожно, DSL!
– Мартин Фаулер
Предметно-ориентированный язык — это
язык программирования с ограниченными
выразительными возможностями,
ориентированный на некую конкретную
предметную область
✤ SQL
✤ REGEXP
✤ TeX/LaTeX
Виды DSL
DSL
ВнутренниеВнешние
✤ PonyORM
✤ WTForm
✤ Django models
Что будет дальше?
✤ Внутренние DSL
✤ Внешние DSL
✤ Инструменты для создания анализаторов
Внутренние DSL
Все возможности
базового языка
Привычный синтаксис
Ограничен базовым
языком
'size': 100MB - 2 * (1MB - 100KB)
GB = lambda x: x*2**30

MB = lambda x: x*2**20

KB = lambda x: x*2**10
GB = 2**30

MB = 2**20

KB = 2**10
config = {

(‘/var/log/nginx/access.log',): {

'size': 100*MB - 2 * (1*MB - 100*KB)
},
(‘/var/log/nginx/error.log',): {

'size': 100*MB - 2 * (1*MB - 100*KB)
}

}

'size': MB(100) - 2*(MB(1) + KB(100))
'size': 100*MB - 2 * (1*MB - 100*KB)
Внешние DSL
Сами выбираем
синтаксис
Необходимо
разрабатывать
анализаторы
compress: true

rules:

- files: [/var/log/nginx/access.log, /var/log/nginx/error.log]

rotate: 5

size: 100MB - 2 * (1MB + 100KB)

postrotate: [/sbin/killall -HUP syslogd]

config = {

'compress': True,

('/var/log/nginx/access.log', '/var/log/nginx/error.log'): {

'rotate': 5,

'size': 100*MB - 2 * (1*MB - 100*KB)
'postrotate': [

'/sbin/killall -HUP syslogd'

]

}

}

Python Lex-Yacc
(PLY)
ply.lex
ply.yacc
ASTRunCode
source
ply.lex
Type Value
PLUS +
MINUS -
MUL *
DIV /
UNIT GB|MB|KB|B
DIGIT d+
LPAREN (
RPAREN )
ply.lex 2 * (1KB + 1KB)
DIGIT = 2
MUL = *
LPAREN = (
DIGIT = 1
UNIT = KB
PLUS = +
DIGIT = 1
UNIT = KB
RPAREN = )
ply.lex 1KB1KB-MB
DIGIT = 1
DIGIT = 1
UNIT = KB
MINUS = -
UNIT = MB
UNIT = KB
ply.yacc
expression : expression PLUS expression

| expression MINUS expression

| expression MUL expression

| expression DIV expression
expression : LPAREN expression RPAREN
expression : DIGIT UNIT
expression : DIGIT
ply.yacc
def p_expression(self, p):

'''

expression : expression PLUS expression

| expression MINUS expression

| expression MUL expression

| expression DIV expression

'''

if p[2] == '+': p[0] = p[1] + p[3]

if p[2] == '-': p[0] = p[1] - p[3]

if p[2] == '*': p[0] = p[1] * p[3]

if p[2] == '/': p[0] = p[1] / p[3]
ply.yacc
precedence = (

('left', 'PLUS', 'MINUS'),

('left', 'MUL', 'DIV'),

)
def p_expression(self, p):

'''

expression : expression PLUS expression

| expression MINUS expression

| expression MUL expression

| expression DIV expression

'''

if p[2] == '+': p[0] = p[1] + p[3]

if p[2] == '-': p[0] = p[1] - p[3]

if p[2] == '*': p[0] = p[1] * p[3]

if p[2] == '/': p[0] = p[1] / p[3]
PLY
Гибкость
Отладка
Обработка ошибок
Понятный код
библиотеки
Высокий порог входа
Многословный
funcparserlib
def tokenize(str):

specs = [

('Spaces', (r'[ strn]+',)),



('PLUS', (r'+',)),

('MINUS', (r'-',)),

('MUL', (r'*',)),

('DIV', (r'/',)),

('UNIT', (r'GB|MB|KB|B',)),

('DIGIT', (r'd+',)),

('LPAREN', (r'(',)),

('RPAREN', (r')',)),

]

return list(filter(

lambda t: t.type != 'Spaces',

(t for t in make_tokenizer(specs)(str))

))
funcparserlib
number = value_of('DIGIT') >> int

unit = number + value_of('UNIT') >> to_bytes

operand = unit | number



makeop = lambda s, f: skip_token(s) >> const(f)

add = makeop('PLUS', operator.add)

sub = makeop('MINUS', operator.sub)

mul = makeop('MUL', operator.mul)

div = makeop('DIV', operator.floordiv)


mul_op = mul | div

add_op = add | sub
funcparserlib
primary = with_forward_decls(

lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN'))

)



term = primary + many(mul_op + primary) >> eval_expr

expr = term + many(add_op + term) >> eval_expr

parser = maybe(expr)
funcparserlib
primary = with_forward_decls(

lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN'))

)



term = primary + many(mul_op + primary) >> eval_expr

expr = term + many(add_op + term) >> eval_expr

parser = maybe(expr)
funcparserlib
2+2*2
primary = with_forward_decls(

lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN'))

)



term = primary + many(mul_op + primary) >> eval_expr

expr = term + many(add_op + term) >> eval_expr

parser = maybe(expr)
funcparserlib
2+2*2
maybe(expr)
primary = with_forward_decls(

lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN'))

)



term = primary + many(mul_op + primary) >> eval_expr

expr = term + many(add_op + term) >> eval_expr

parser = maybe(expr)
funcparserlib
2+2*2
term
primary = with_forward_decls(

lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN'))

)



term = primary + many(mul_op + primary) >> eval_expr

expr = term + many(add_op + term) >> eval_expr

parser = maybe(expr)
funcparserlib
2+2*2
primary
primary = with_forward_decls(

lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN'))

)



term = primary + many(mul_op + primary) >> eval_expr

expr = term + many(add_op + term) >> eval_expr

parser = maybe(expr)
funcparserlib
2+2*22
operand
primary = with_forward_decls(

lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN'))

)



term = primary + many(mul_op + primary) >> eval_expr

expr = term + many(add_op + term) >> eval_expr

parser = maybe(expr)
funcparserlib
2+2*22
many(mul_op + primary)
primary = with_forward_decls(

lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN'))

)



term = primary + many(mul_op + primary) >> eval_expr

expr = term + many(add_op + term) >> eval_expr

parser = maybe(expr)
funcparserlib
many(add_op + term)
2+2*22+
primary = with_forward_decls(

lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN'))

)



term = primary + many(mul_op + primary) >> eval_expr

expr = term + many(add_op + term) >> eval_expr

parser = maybe(expr)
funcparserlib
primary
2+2*22+
primary = with_forward_decls(

lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN'))

)



term = primary + many(mul_op + primary) >> eval_expr

expr = term + many(add_op + term) >> eval_expr

parser = maybe(expr)
funcparserlib
operand
2+2*22+2
primary = with_forward_decls(

lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN'))

)



term = primary + many(mul_op + primary) >> eval_expr

expr = term + many(add_op + term) >> eval_expr

parser = maybe(expr)
funcparserlib
2+2*22+2*
many(mul_op + primary)
primary = with_forward_decls(

lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN'))

)



term = primary + many(mul_op + primary) >> eval_expr

expr = term + many(add_op + term) >> eval_expr

parser = maybe(expr)
funcparserlib
operand
2+2*22+2*2
primary = with_forward_decls(

lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN'))

)



term = primary + many(mul_op + primary) >> eval_expr

expr = term + many(add_op + term) >> eval_expr

parser = maybe(expr)
funcparserlib
eval_expr
2+2*22+2*2
2+4
primary = with_forward_decls(

lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN'))

)



term = primary + many(mul_op + primary) >> eval_expr

expr = term + many(add_op + term) >> eval_expr

parser = maybe(expr)
funcparserlib
eval_expr
6
primary = with_forward_decls(

lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN'))

)



term = primary + many(mul_op + primary) >> eval_expr

expr = term + many(add_op + term) >> eval_expr

parser = maybe(expr)
funcparserlib
maybe(expr)
funcparserlib
Компактный
Гибкий
Для любителей
функционального
программирования :)
Многое приходится
делать руками
Для любителей
функционального
программирования :)
pyparsing
pyparsing
plusop = oneOf('+ -')

multop = oneOf('* /')



digit = Word(nums)

unit = digit + oneOf('GB MB KB’)
operand = unit | digit



parser = Forward()

primary = operand | Literal('(') + parser + Literal(')')

term = (primary + ZeroOrMore(multop + primary)).setParseAction(eval_expr)

expr = (term + ZeroOrMore(plusop + term)).setParseAction(eval_expr)

parser << Optional(expr)

pyparsing
plusop = oneOf('+ -')

multop = oneOf('* /')

digit = Word(nums)

unit = digit + oneOf('GB MB KB’)
operand = unit | digit


parser = operatorPrecedence(

operand, [

(multop, 2, opAssoc.LEFT, calculate),

(plusop, 2, opAssoc.LEFT, calculate)

]

)
pyparsing
Понятный код
Базовые компоненты
Свои компоненты
Документация
Отладка
А что насчет быстродействия?
Скорость
Скорость
А что же пользователи?
Сообщения об ошибках
Traceback (most recent call last):
File "size_parser/on_ply.py", line 100, in parse
return parser.parse(string)
File "size_parser/on_ply.py", line 94, in parse
p = self._parser.parse(data, debug=self._debug)
File ".env/site-packages/ply/yacc.py", line 331, in parse
return self.parseopt_notrack(input, lexer, debug, tracking, tokenfunc)
File ".env/site-packages/ply/yacc.py", line 1049, in parseopt_notrack
lookahead = get_token() # Get the next token
File ".env/site-packages/ply/lex.py", line 396, in token
raise LexError("Illegal character '%s' at index %d" % (lexdata[lexpos], lexpos), lexdata[lexpos:])
ply.lex.LexError: Illegal character '_' at index 5
Traceback (most recent call last):
File "size_parser/on_ply.py", line 100, in parse
return parser.parse(string)
File "size_parser/on_ply.py", line 94, in parse
p = self._parser.parse(data, debug=self._debug)
File ".env/site-packages/ply/yacc.py", line 331, in parse
return self.parseopt_notrack(input, lexer, debug, tracking, tokenfunc)
File ".env/site-packages/ply/yacc.py", line 1049, in parseopt_notrack
lookahead = get_token() # Get the next token
File ".env/site-packages/ply/lex.py", line 396, in token
raise LexError("Illegal character '%s' at index %d" % (lexdata[lexpos], lexpos), lexdata[lexpos:])
ply.lex.LexError: Illegal character '_' at index 5
Сообщения об ошибках
ply


def t_error(self, t):

raise ParserException(t, self.source)



def p_error(self, p):

raise ParserException(p, self.source)
>>> parse(‘1MB+_GB-100KB’)
Unexpected "_GB-100KB" at position 4:
1MB+_GB-100KB
^^^
Так что же выбрать?
Что же выбрать?
pyparsing
✤ Хочу легко все описать
✤ Быстродействие не главное
Что же выбрать?
funcparserlib
pyparsing
✤ Хочу легко все описать
✤ Быстродействие не главное
✤ Люблю функциональное
программирование
✤ Быстродействие не главное
Что же выбрать?
PLY
funcparserlib
pyparsing
✤ Хочу как в учебнике
✤ Скорость работы - главное!
✤ Хочу легко все описать
✤ Быстродействие не главное
✤ Люблю функциональное
программирование
✤ Быстродействие не главное
И что в итоге?
✤ Для простых задач попробуйте:
✤ Средства самого языка
✤ Регулярные выражения
✤ Если задача сложная:
✤ Внутренние DSL
✤ Yaml, Json, XML, …
✤ Внешние DSL
Спасибо за внимание!
Вопросы?
http://tsyganov-ivan.com/

More Related Content

What's hot

An introduction to functional programming with Go [redux]
An introduction to functional programming with Go [redux]An introduction to functional programming with Go [redux]
An introduction to functional programming with Go [redux]Eleanor McHugh
 
Node.JS
Node.JSNode.JS
Node.JSeibaan
 
Programación con python
Programación con pythonProgramación con python
Programación con pythonu2014231724
 
Boost.勉強会#4 Boost.Proto
Boost.勉強会#4 Boost.ProtoBoost.勉強会#4 Boost.Proto
Boost.勉強会#4 Boost.Protofjnl
 
Anatomy of the loadable kernel module (lkm)
Anatomy of the loadable kernel module (lkm)Anatomy of the loadable kernel module (lkm)
Anatomy of the loadable kernel module (lkm)Adrian Huang
 
Rambler.iOS #8: Чистые unit-тесты
Rambler.iOS #8: Чистые unit-тестыRambler.iOS #8: Чистые unit-тесты
Rambler.iOS #8: Чистые unit-тестыRAMBLER&Co
 
JS programowanie obiektowe
JS  programowanie obiektoweJS  programowanie obiektowe
JS programowanie obiektowePiotr Czajkowski
 
Java Script - Object-Oriented Programming
Java Script - Object-Oriented ProgrammingJava Script - Object-Oriented Programming
Java Script - Object-Oriented Programmingintive
 
数式を構文解析した話
数式を構文解析した話数式を構文解析した話
数式を構文解析した話y1r96 Ueno
 
Тененёв Анатолий, Boost.Asio в алгоритмической торговле
Тененёв Анатолий, Boost.Asio в алгоритмической торговлеТененёв Анатолий, Boost.Asio в алгоритмической торговле
Тененёв Анатолий, Boost.Asio в алгоритмической торговлеPlatonov Sergey
 
BABELで、ES2015(ES6)を学ぼう!
BABELで、ES2015(ES6)を学ぼう!BABELで、ES2015(ES6)を学ぼう!
BABELで、ES2015(ES6)を学ぼう!Toshio Ehara
 
Максим cdump Андреев, WPAD, Security Meetup 9 апреля, Mail.Ru Group
Максим cdump Андреев, WPAD, Security Meetup 9 апреля, Mail.Ru GroupМаксим cdump Андреев, WPAD, Security Meetup 9 апреля, Mail.Ru Group
Максим cdump Андреев, WPAD, Security Meetup 9 апреля, Mail.Ru GroupMail.ru Group
 
Tugas pemrograman jaringan
Tugas pemrograman jaringanTugas pemrograman jaringan
Tugas pemrograman jaringanBanser Sahara
 
Aller plus loin avec Doctrine2
Aller plus loin avec Doctrine2Aller plus loin avec Doctrine2
Aller plus loin avec Doctrine2André Tapia
 
Docstringを書こう!
Docstringを書こう!Docstringを書こう!
Docstringを書こう!cocodrips
 

What's hot (20)

An introduction to functional programming with Go [redux]
An introduction to functional programming with Go [redux]An introduction to functional programming with Go [redux]
An introduction to functional programming with Go [redux]
 
JQuery
JQueryJQuery
JQuery
 
Node.JS
Node.JSNode.JS
Node.JS
 
Programación con python
Programación con pythonProgramación con python
Programación con python
 
Zadatak
ZadatakZadatak
Zadatak
 
Boost.勉強会#4 Boost.Proto
Boost.勉強会#4 Boost.ProtoBoost.勉強会#4 Boost.Proto
Boost.勉強会#4 Boost.Proto
 
Anatomy of the loadable kernel module (lkm)
Anatomy of the loadable kernel module (lkm)Anatomy of the loadable kernel module (lkm)
Anatomy of the loadable kernel module (lkm)
 
Rambler.iOS #8: Чистые unit-тесты
Rambler.iOS #8: Чистые unit-тестыRambler.iOS #8: Чистые unit-тесты
Rambler.iOS #8: Чистые unit-тесты
 
Librerias de c++
Librerias de c++Librerias de c++
Librerias de c++
 
Mikstura it2013
Mikstura it2013Mikstura it2013
Mikstura it2013
 
JS programowanie obiektowe
JS  programowanie obiektoweJS  programowanie obiektowe
JS programowanie obiektowe
 
Java Script - Object-Oriented Programming
Java Script - Object-Oriented ProgrammingJava Script - Object-Oriented Programming
Java Script - Object-Oriented Programming
 
数式を構文解析した話
数式を構文解析した話数式を構文解析した話
数式を構文解析した話
 
Тененёв Анатолий, Boost.Asio в алгоритмической торговле
Тененёв Анатолий, Boost.Asio в алгоритмической торговлеТененёв Анатолий, Boost.Asio в алгоритмической торговле
Тененёв Анатолий, Boost.Asio в алгоритмической торговле
 
Git a starter kit
Git a starter kitGit a starter kit
Git a starter kit
 
BABELで、ES2015(ES6)を学ぼう!
BABELで、ES2015(ES6)を学ぼう!BABELで、ES2015(ES6)を学ぼう!
BABELで、ES2015(ES6)を学ぼう!
 
Максим cdump Андреев, WPAD, Security Meetup 9 апреля, Mail.Ru Group
Максим cdump Андреев, WPAD, Security Meetup 9 апреля, Mail.Ru GroupМаксим cdump Андреев, WPAD, Security Meetup 9 апреля, Mail.Ru Group
Максим cdump Андреев, WPAD, Security Meetup 9 апреля, Mail.Ru Group
 
Tugas pemrograman jaringan
Tugas pemrograman jaringanTugas pemrograman jaringan
Tugas pemrograman jaringan
 
Aller plus loin avec Doctrine2
Aller plus loin avec Doctrine2Aller plus loin avec Doctrine2
Aller plus loin avec Doctrine2
 
Docstringを書こう!
Docstringを書こう!Docstringを書こう!
Docstringを書こう!
 

PyconRu 2016. Осторожно, DSL!

  • 2. server { location / { proxy_pass http://localhost:8080/; } location ~ .(gif|jpg|png)$ { root /data/images; } }
  • 3. version: '2' services: db: image: postgres web: build: . command: python manage.py runserver 0.0.0.0:8000 volumes: - .:/code ports: - "8000:8000" depends_on: - db DOCKER COMPOSE
  • 4. LOGROTATE compress "/var/log/httpd/access.log" /var/log/httpd/error.log { rotate 5 size=100k sharedscripts nocompress postrotate /sbin/killall -HUP httpd endscript } /var/log/messages { rotate 5 weekly postrotate /sbin/killall -HUP syslogd endscript }
  • 5. compress "/var/log/httpd/access.log" /var/log/httpd/error.log { rotate 5 size=100k sharedscripts nocompress postrotate /sbin/killall -HUP httpd endscript }
  • 6. config = {
 'compress': True,
 ('/var/log/nginx/access.log', '/var/log/nginx/error.log'): {
 'rotate': 5,
 'size': 102400, 'compress': False,
 'sharedscripts': True,
 'postrotate': [
 '/sbin/killall -HUP syslogd'
 ]
 }
 }
 compress "/var/log/httpd/access.log" /var/log/httpd/error.log { rotate 5 size=100k sharedscripts nocompress postrotate /sbin/killall -HUP httpd endscript }
  • 7. config = {
 'compress': True,
 ('/var/log/nginx/access.log', '/var/log/nginx/error.log'): {
 'rotate': 5,
 'size': 102400, 'compress': False,
 'sharedscripts': True,
 'postrotate': [
 '/sbin/killall -HUP syslogd'
 ]
 }
 }
 compress "/var/log/httpd/access.log" /var/log/httpd/error.log { rotate 5 size=100k sharedscripts nocompress postrotate /sbin/killall -HUP httpd endscript } size=100k 'size': 102400,
  • 8. config = {
 'compress': True,
 ('/var/log/nginx/access.log', '/var/log/nginx/error.log'): {
 'rotate': 5,
 'size': 102400, 'postrotate': [
 '/sbin/killall -HUP syslogd'
 ]
 }
 }
 'size': '100 KB',
  • 9. config = {
 'compress': True,
 ('/var/log/nginx/access.log', '/var/log/nginx/error.log'): {
 'rotate': 5,
 'size': 102400, 'postrotate': [
 '/sbin/killall -HUP syslogd'
 ]
 }
 }
 'size': '100 KB', def get_size(self, value):
 size_value, _, size_unit = value.partition(' ')
 return self.units_to_bytes(size_value, size_unit)
  • 10. config = {
 'compress': True,
 ('/var/log/nginx/access.log', '/var/log/nginx/error.log'): {
 'rotate': 5,
 'size': 102400, 'postrotate': [
 '/sbin/killall -HUP syslogd'
 ]
 }
 }
 'size': '1MB + 100KB',
  • 11. config = {
 'compress': True,
 ('/var/log/nginx/access.log', '/var/log/nginx/error.log'): {
 'rotate': 5,
 'size': 102400, 'postrotate': [
 '/sbin/killall -HUP syslogd'
 ]
 }
 }
 'size': '1MB + 100KB', parse_size = re.compile(
 pattern=r'''
 (?P<Value>d+)s*
 (?P<ValueUnit>KB|MB|GB)?s*
 ((?P<Operator>[-+/*])s*)?
 (
 (?P<Delta>d+)s*
 (?P<DeltaUnit>KB|MB|GB)?
 )? ''',
 flags=re.IGNORECASE | re.VERBOSE
 ).search
  • 12. config = {
 'compress': True,
 ('/var/log/nginx/access.log', '/var/log/nginx/error.log'): {
 'rotate': 5,
 'size': 102400, 'postrotate': [
 '/sbin/killall -HUP syslogd'
 ]
 }
 }
 'size': ‘100MB - 2 * (1MB + 100KB)’,
  • 13. Some people, when confronted with a problem, think "I know, I’ll use regular expressions." Now they have two problems. — Jamie Zawinski
  • 15. – Мартин Фаулер Предметно-ориентированный язык — это язык программирования с ограниченными выразительными возможностями, ориентированный на некую конкретную предметную область
  • 16. ✤ SQL ✤ REGEXP ✤ TeX/LaTeX Виды DSL DSL ВнутренниеВнешние ✤ PonyORM ✤ WTForm ✤ Django models
  • 17. Что будет дальше? ✤ Внутренние DSL ✤ Внешние DSL ✤ Инструменты для создания анализаторов
  • 18. Внутренние DSL Все возможности базового языка Привычный синтаксис Ограничен базовым языком 'size': 100MB - 2 * (1MB - 100KB)
  • 19. GB = lambda x: x*2**30
 MB = lambda x: x*2**20
 KB = lambda x: x*2**10 GB = 2**30
 MB = 2**20
 KB = 2**10 config = {
 (‘/var/log/nginx/access.log',): {
 'size': 100*MB - 2 * (1*MB - 100*KB) }, (‘/var/log/nginx/error.log',): {
 'size': 100*MB - 2 * (1*MB - 100*KB) }
 }
 'size': MB(100) - 2*(MB(1) + KB(100)) 'size': 100*MB - 2 * (1*MB - 100*KB)
  • 21. compress: true
 rules:
 - files: [/var/log/nginx/access.log, /var/log/nginx/error.log]
 rotate: 5
 size: 100MB - 2 * (1MB + 100KB)
 postrotate: [/sbin/killall -HUP syslogd]
 config = {
 'compress': True,
 ('/var/log/nginx/access.log', '/var/log/nginx/error.log'): {
 'rotate': 5,
 'size': 100*MB - 2 * (1*MB - 100*KB) 'postrotate': [
 '/sbin/killall -HUP syslogd'
 ]
 }
 }

  • 24. ply.lex Type Value PLUS + MINUS - MUL * DIV / UNIT GB|MB|KB|B DIGIT d+ LPAREN ( RPAREN )
  • 25. ply.lex 2 * (1KB + 1KB) DIGIT = 2 MUL = * LPAREN = ( DIGIT = 1 UNIT = KB PLUS = + DIGIT = 1 UNIT = KB RPAREN = )
  • 26. ply.lex 1KB1KB-MB DIGIT = 1 DIGIT = 1 UNIT = KB MINUS = - UNIT = MB UNIT = KB
  • 27. ply.yacc expression : expression PLUS expression
 | expression MINUS expression
 | expression MUL expression
 | expression DIV expression expression : LPAREN expression RPAREN expression : DIGIT UNIT expression : DIGIT
  • 28. ply.yacc def p_expression(self, p):
 '''
 expression : expression PLUS expression
 | expression MINUS expression
 | expression MUL expression
 | expression DIV expression
 '''
 if p[2] == '+': p[0] = p[1] + p[3]
 if p[2] == '-': p[0] = p[1] - p[3]
 if p[2] == '*': p[0] = p[1] * p[3]
 if p[2] == '/': p[0] = p[1] / p[3]
  • 29. ply.yacc precedence = (
 ('left', 'PLUS', 'MINUS'),
 ('left', 'MUL', 'DIV'),
 ) def p_expression(self, p):
 '''
 expression : expression PLUS expression
 | expression MINUS expression
 | expression MUL expression
 | expression DIV expression
 '''
 if p[2] == '+': p[0] = p[1] + p[3]
 if p[2] == '-': p[0] = p[1] - p[3]
 if p[2] == '*': p[0] = p[1] * p[3]
 if p[2] == '/': p[0] = p[1] / p[3]
  • 32. def tokenize(str):
 specs = [
 ('Spaces', (r'[ strn]+',)),
 
 ('PLUS', (r'+',)),
 ('MINUS', (r'-',)),
 ('MUL', (r'*',)),
 ('DIV', (r'/',)),
 ('UNIT', (r'GB|MB|KB|B',)),
 ('DIGIT', (r'd+',)),
 ('LPAREN', (r'(',)),
 ('RPAREN', (r')',)),
 ]
 return list(filter(
 lambda t: t.type != 'Spaces',
 (t for t in make_tokenizer(specs)(str))
 )) funcparserlib
  • 33. number = value_of('DIGIT') >> int
 unit = number + value_of('UNIT') >> to_bytes
 operand = unit | number
 
 makeop = lambda s, f: skip_token(s) >> const(f)
 add = makeop('PLUS', operator.add)
 sub = makeop('MINUS', operator.sub)
 mul = makeop('MUL', operator.mul)
 div = makeop('DIV', operator.floordiv) 
 mul_op = mul | div
 add_op = add | sub funcparserlib
  • 34. primary = with_forward_decls(
 lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN'))
 )
 
 term = primary + many(mul_op + primary) >> eval_expr
 expr = term + many(add_op + term) >> eval_expr
 parser = maybe(expr) funcparserlib
  • 35. primary = with_forward_decls(
 lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN'))
 )
 
 term = primary + many(mul_op + primary) >> eval_expr
 expr = term + many(add_op + term) >> eval_expr
 parser = maybe(expr) funcparserlib 2+2*2
  • 36. primary = with_forward_decls(
 lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN'))
 )
 
 term = primary + many(mul_op + primary) >> eval_expr
 expr = term + many(add_op + term) >> eval_expr
 parser = maybe(expr) funcparserlib 2+2*2 maybe(expr)
  • 37. primary = with_forward_decls(
 lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN'))
 )
 
 term = primary + many(mul_op + primary) >> eval_expr
 expr = term + many(add_op + term) >> eval_expr
 parser = maybe(expr) funcparserlib 2+2*2 term
  • 38. primary = with_forward_decls(
 lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN'))
 )
 
 term = primary + many(mul_op + primary) >> eval_expr
 expr = term + many(add_op + term) >> eval_expr
 parser = maybe(expr) funcparserlib 2+2*2 primary
  • 39. primary = with_forward_decls(
 lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN'))
 )
 
 term = primary + many(mul_op + primary) >> eval_expr
 expr = term + many(add_op + term) >> eval_expr
 parser = maybe(expr) funcparserlib 2+2*22 operand
  • 40. primary = with_forward_decls(
 lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN'))
 )
 
 term = primary + many(mul_op + primary) >> eval_expr
 expr = term + many(add_op + term) >> eval_expr
 parser = maybe(expr) funcparserlib 2+2*22 many(mul_op + primary)
  • 41. primary = with_forward_decls(
 lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN'))
 )
 
 term = primary + many(mul_op + primary) >> eval_expr
 expr = term + many(add_op + term) >> eval_expr
 parser = maybe(expr) funcparserlib many(add_op + term) 2+2*22+
  • 42. primary = with_forward_decls(
 lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN'))
 )
 
 term = primary + many(mul_op + primary) >> eval_expr
 expr = term + many(add_op + term) >> eval_expr
 parser = maybe(expr) funcparserlib primary 2+2*22+
  • 43. primary = with_forward_decls(
 lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN'))
 )
 
 term = primary + many(mul_op + primary) >> eval_expr
 expr = term + many(add_op + term) >> eval_expr
 parser = maybe(expr) funcparserlib operand 2+2*22+2
  • 44. primary = with_forward_decls(
 lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN'))
 )
 
 term = primary + many(mul_op + primary) >> eval_expr
 expr = term + many(add_op + term) >> eval_expr
 parser = maybe(expr) funcparserlib 2+2*22+2* many(mul_op + primary)
  • 45. primary = with_forward_decls(
 lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN'))
 )
 
 term = primary + many(mul_op + primary) >> eval_expr
 expr = term + many(add_op + term) >> eval_expr
 parser = maybe(expr) funcparserlib operand 2+2*22+2*2
  • 46. primary = with_forward_decls(
 lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN'))
 )
 
 term = primary + many(mul_op + primary) >> eval_expr
 expr = term + many(add_op + term) >> eval_expr
 parser = maybe(expr) funcparserlib eval_expr 2+2*22+2*2
  • 47. 2+4 primary = with_forward_decls(
 lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN'))
 )
 
 term = primary + many(mul_op + primary) >> eval_expr
 expr = term + many(add_op + term) >> eval_expr
 parser = maybe(expr) funcparserlib eval_expr
  • 48. 6 primary = with_forward_decls(
 lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN'))
 )
 
 term = primary + many(mul_op + primary) >> eval_expr
 expr = term + many(add_op + term) >> eval_expr
 parser = maybe(expr) funcparserlib maybe(expr)
  • 49. funcparserlib Компактный Гибкий Для любителей функционального программирования :) Многое приходится делать руками Для любителей функционального программирования :)
  • 51. pyparsing plusop = oneOf('+ -')
 multop = oneOf('* /')
 
 digit = Word(nums)
 unit = digit + oneOf('GB MB KB’) operand = unit | digit
 
 parser = Forward()
 primary = operand | Literal('(') + parser + Literal(')')
 term = (primary + ZeroOrMore(multop + primary)).setParseAction(eval_expr)
 expr = (term + ZeroOrMore(plusop + term)).setParseAction(eval_expr)
 parser << Optional(expr)

  • 52. pyparsing plusop = oneOf('+ -')
 multop = oneOf('* /')
 digit = Word(nums)
 unit = digit + oneOf('GB MB KB’) operand = unit | digit 
 parser = operatorPrecedence(
 operand, [
 (multop, 2, opAssoc.LEFT, calculate),
 (plusop, 2, opAssoc.LEFT, calculate)
 ]
 )
  • 53. pyparsing Понятный код Базовые компоненты Свои компоненты Документация Отладка
  • 54. А что насчет быстродействия?
  • 57. А что же пользователи?
  • 58. Сообщения об ошибках Traceback (most recent call last): File "size_parser/on_ply.py", line 100, in parse return parser.parse(string) File "size_parser/on_ply.py", line 94, in parse p = self._parser.parse(data, debug=self._debug) File ".env/site-packages/ply/yacc.py", line 331, in parse return self.parseopt_notrack(input, lexer, debug, tracking, tokenfunc) File ".env/site-packages/ply/yacc.py", line 1049, in parseopt_notrack lookahead = get_token() # Get the next token File ".env/site-packages/ply/lex.py", line 396, in token raise LexError("Illegal character '%s' at index %d" % (lexdata[lexpos], lexpos), lexdata[lexpos:]) ply.lex.LexError: Illegal character '_' at index 5
  • 59. Traceback (most recent call last): File "size_parser/on_ply.py", line 100, in parse return parser.parse(string) File "size_parser/on_ply.py", line 94, in parse p = self._parser.parse(data, debug=self._debug) File ".env/site-packages/ply/yacc.py", line 331, in parse return self.parseopt_notrack(input, lexer, debug, tracking, tokenfunc) File ".env/site-packages/ply/yacc.py", line 1049, in parseopt_notrack lookahead = get_token() # Get the next token File ".env/site-packages/ply/lex.py", line 396, in token raise LexError("Illegal character '%s' at index %d" % (lexdata[lexpos], lexpos), lexdata[lexpos:]) ply.lex.LexError: Illegal character '_' at index 5 Сообщения об ошибках
  • 60. ply 
 def t_error(self, t):
 raise ParserException(t, self.source)
 
 def p_error(self, p):
 raise ParserException(p, self.source) >>> parse(‘1MB+_GB-100KB’) Unexpected "_GB-100KB" at position 4: 1MB+_GB-100KB ^^^
  • 61. Так что же выбрать?
  • 62. Что же выбрать? pyparsing ✤ Хочу легко все описать ✤ Быстродействие не главное
  • 63. Что же выбрать? funcparserlib pyparsing ✤ Хочу легко все описать ✤ Быстродействие не главное ✤ Люблю функциональное программирование ✤ Быстродействие не главное
  • 64. Что же выбрать? PLY funcparserlib pyparsing ✤ Хочу как в учебнике ✤ Скорость работы - главное! ✤ Хочу легко все описать ✤ Быстродействие не главное ✤ Люблю функциональное программирование ✤ Быстродействие не главное
  • 65. И что в итоге? ✤ Для простых задач попробуйте: ✤ Средства самого языка ✤ Регулярные выражения ✤ Если задача сложная: ✤ Внутренние DSL ✤ Yaml, Json, XML, … ✤ Внешние DSL