16. ERB Ruby Scanner
class TrimScanner < Scanner # :nodoc:
def initialize(src, trim_mode, percent)
super
@trim_mode = trim_mode
@percent = percent
if @trim_mode == '>'
@scan_line = self.method(:trim_line1)
elsif @trim_mode == '<>'
@scan_line = self.method(:trim_line2)
elsif @trim_mode == '-'
@scan_line = self.method(:explicit_trim_line)
else
@scan_line = self.method(:scan_line)
end
end
attr_accessor :stag
def scan(&block)
@stag = nil
if @percent
@src.each_line do |line|
percent_line(line, &block)
end
else
@scan_line.call(@src, &block)
end
nil
end
def percent_line(line, &block)
if @stag || line[0] != ?%
return @scan_line.call(line, &block)
end
line[0] = ''
if line[0] == ?%
@scan_line.call(line, &block)
else
yield(PercentLine.new(line.chomp))
end
end
17. def trim_line2(line)
head = nil
line.scan(/(.*?)(<%%|%%>|<%=|<%#|<%|%>n|%>|n|z)/m) do |tokens|
tokens.each do |token|
next if token.empty?
head = token unless head
ERB Ruby Scanner
if token == "%>n"
yield('%>')
if is_erb_stag?(head)
yield(:cr)
else
yield("n")
end
head = nil
else
yield(token)
head = nil if token == "n"
end
end
end
end
def explicit_trim_line(line)
line.scan(/(.*?)(^[ t]*<%-|<%-|<%%|%%>|<%=|<%#|<%|-%>n|-%>|%>|z)/m) do |tokens|
tokens.each do |token|
next if token.empty?
if @stag.nil? && /[ t]*<%-/ =~ token
yield('<%')
elsif @stag && token == "-%>n"
yield('%>')
yield(:cr)
elsif @stag && token == '-%>'
yield('%>')
else
yield(token)
end
end
end
end
ERB_STAG = %w(<%= <%# <%)
def is_erb_stag?(s)
ERB_STAG.member?(s)
end
end
Scanner.default_scanner = TrimScanner
Scanner
18. Racc Ruby Scanner
def scan_action
buf = ''
nest = 1
pre = nil
@in_block = 'action'
begin
pre = nil
if s = reads(/As+/)
# does not set 'pre'
buf << s
end
until @line.empty?
if s = reads(/A[^'"`{}%#/$]+/)
buf << (pre = s)
next
end
case ch = read(1)
when '{'
nest += 1
buf << (pre = ch)
when '}'
nest -= 1
if nest == 0
@in_block = nil
return buf
end
buf << (pre = ch)
when '#' # comment
buf << ch << @line
break
when "'", '"', '`'
buf << (pre = scan_quoted(ch))
when '%'
if literal_head? pre, @line
# % string, regexp, array
buf << ch
case ch = read(1)
when /[qQx]/n
buf << ch << (pre = scan_quoted(read(1), '%string'))
when /wW/n
19. # % string, regexp, array
buf << ch
case ch = read(1)
when /[qQx]/n
buf << ch << (pre = scan_quoted(read(1), '%string'))
Racc Ruby Scanner
when /wW/n
buf << ch << (pre = scan_quoted(read(1), '%array'))
when /s/n
buf << ch << (pre = scan_quoted(read(1), '%symbol'))
when /r/n
buf << ch << (pre = scan_quoted(read(1), '%regexp'))
when /[a-zA-Z0-9= ]/n # does not include "_"
scan_error! "unknown type of % literal '%#{ch}'"
else
buf << (pre = scan_quoted(ch, '%string'))
end
else
# operator
buf << '||op->' if $raccs_print_type
buf << (pre = ch)
end
when '/'
if literal_head? pre, @line
# regexp
buf << (pre = scan_quoted(ch, 'regexp'))
else
# operator
buf << '||op->' if $raccs_print_type
buf << (pre = ch)
end
when '$' # gvar
buf << ch << (pre = read(1))
else
raise 'racc: fatal: must not happen'
end
end
buf << "n"
end while next_line()
raise 'racc: fatal: scan finished before parser finished'
end
22. Ripper
• test/ripper/test_files.rb
class Parser < Ripper
PARSER_EVENTS.each {|n| eval "def on_#{n}(*args) r = [:#{n}, *args]; r.inspect; Object.new end" }
SCANNER_EVENTS.each {|n| eval "def on_#{n}(*args) r = [:#{n}, *args]; r.inspect; Object.new end" }
end
•
23. Ripper Scanner
class RubyCodeDetector < Ripper
class ParseError < Exception
attr_reader :message,:result
def initialize(r,msg)
@result = r
@message = msg
end
end
def on_parse_error(msg)
l = @lines[lineno-1][0,column-1]
src = (@lines[0,lineno-1]+[l])*"n"
src.force_encoding(@src.encoding)
raise ParseError.new(src,msg)
end
def initialize(*args)
@src=args[0]
@lines=@src.split(/n/).map{|i| i.force_encoding("ASCII-8BIT") }
super
end
def parse
super
raise ParseError.new(@src,"syntax error, unexpected $end, expecting '}'")
nil
rescue ParseError => e
if e.message =~ /G syntax error, unexpected '}'/
e.result
else
raise $!
nil
end
end
end
24. Ripper Scanner
class RubyCodeDetector < Ripper
class ParseError < Exception
attr_reader :message,:result
def initialize(r,msg)
@result = r
@message = msg
end
end
def on_parse_error(msg)
l = @lines[lineno-1][0,column-1]
src = (@lines[0,lineno-1]+[l])*"n"
src.force_encoding(@src.encoding)
raise ParseError.new(src,msg)
end
def initialize(*args)
@src=args[0]
@lines=@src.split(/n/).map{|i| i.force_encoding("ASCII-8BIT") }
super
end
def parse
super
raise ParseError.new(@src,"syntax error, unexpected $end, expecting '}'")
nil
rescue ParseError => e
if e.message =~ /G syntax error, unexpected '}'/
e.result
else
raise $!
nil
end
end
end
