RubyConf Brazil 2010

OMG HELLO!
Friday, November 5, 2010

THANK YOU

Aaron Patterson

AT&T, AT&T logo and all AT&T related marks are trademarks of AT&T Intellectual Property and/or AT&T affiliated companies.

@tenderlove

nokogiri

ruby core
committer

Psych

rails core
committer

ActiveRecord

anywhere...

aaron.patterson@gmail.com

WWFMD?

Preparations
the @tenderlove way

What should
I talk about?

Yehuda?

Evan?

I will talk
about
Concurrency!

Threads!

Iterators!

Generators!

Fibers!

Coroutines!

the @tenderlove
process for
producing slides

BOOKS ARE IN
STORES NOW!

ONLY $24.99!!!

BUY TODAY!

Fun with Fibers!

Concurrency

WARNING!

fork()

fork()
foo = "hello"
fork do
foo << " world"
puts foo
end
fork do
foo << " everyone!"
puts foo
end
puts foo
Process.wait

Copy on Write

Threads

Why Use Threads?

Thread API

thread = Thread.new {
... do some work ...
}
thread.join
new and join

100.times { Thread.new { ... } }
(Thread.list - [Thread.main]).each { |t|
t.join
}
list / main

Return Value
thread = Thread.new do
10
end
thread.join
thread.value # => 10

Shared Data

n = 0
def increment(v)
v + 1
end
threads = 2.times.map {
Thread.new {
1_000_000.times { n = increment(n) }
}
}
threads.each { |x| x.join }
puts n

2,000,000?

Read
Increment
Update
v = n
v += 1
n = v
n = 2
n = 2
n = 3

Thread 1 Thread 2

Read
Read
Thread 1 Thread 2
Increment
Increment
Update
Update

Read
Read
Thread 1 Thread 2
Increment
Increment
Update
Update
n = 2
n = 3

Solutions

Locking

Mutex
n = 0
mutex = Mutex.new
def increment(v)
v + 1
end
2.times do
Thread.new {
1_000_000.times {
mutex.synchronize { n = increment(n) }
}
}
end

Mutex_m
require 'mutex_m'
class Tenderlove
include Mutex_m
end
n = 0
mutex = Tenderlove.new
mutex.synchronize { ... }

Conditional Variable

mutex = Mutex.new
resource = ConditionVariable.new
a = Thread.new {
mutex.synchronize {
puts "hello"
resource.wait(mutex)
puts "world"
}
}
b = Thread.new {
mutex.synchronize {
puts "hello1"
sleep 1
resource.signal
}
}
a.join
b.join

Thread Safe
Objects

Queue
queue = Queue.new
[
Thread.new { 100.times {
queue << "omg!"
} },
Thread.new { 100.times {
queue << "omgagain!"
} },
].each { |t| t.join }

SizedQueue

Iterators

Internal

%w{ a b c d e }.each do
...
...
end

External

list = %w{ a b }
iter = Enumerator.new(list)
iter.next # => a
iter.next # => b
iter.next # raises StopIteration

list = %w{ a b }
begin
3.times do
iter.next
end
rescue StopIteration => e
p e.result
end

list = %w{ a b }
begin
3.times do
iter.next
end
rescue StopIteration => e
p e.result # => ["a", "b"]
end

External => Internal

list = %w{ a b }
iter.each do |thing|
p thing
end

Specify a Method

list = %w{ a b }
iter = Enumerator.new(list, :map)
x = iter.each do |thing|
thing + " hello!"
end
p x

list = %w{ a b }
x = iter.each do |thing|
thing + " hello!"
end
p x # => ["a hello!", "b hello!"]

list = %w{ a b }
x = iter.each_with_index do |thing, i|
thing + ":#{i}"
end
p x

list = %w{ a b }
x = iter.each_with_index do |thing, i|
thing + ":#{i}"
end
p x # => ["a:0", "b:1"]

DRY it up

list = %w{ a b }
iter = list.enum_for(:each)
3.times do
p iter.next
end

list = %w{ a b }
iter = list.each
3.times do
p itr.next
end

list = %w{ a b }
list.map.each_with_index { |obj, i|
"#{obj} => #{i}"
}

list = %w{ a b }
list.map.each_with_index { |obj, i|
"#{obj} => #{i}"
} # => ["a => 0", "b => 1"]

Fun with Iterators!

Parallel Iteration

a = %w{ a b c d }
b = %w{ a b c d }
iter = a.each
b.all? { |v| iter.next == v }

Iteration Strategy

Tree
Node
NodeNode
NodeNode

Node
class Node < Struct.new(:children)
def initialize(children = [])
super
end
end

Bottom Up
class DepthFirst < Struct.new(:tree)
def each(&block)
iterate(tree, block)
end
private
def iterate(node, block)
node.children.each { |n|
iterate(n, block)
}
block.call node
end
end

Node
NodeNode
NodeNode
Bottom Up
Node
NodeNode
NodeNode

Top Down
class TopDown < Struct.new(:tree)
def each(&block)
iterate(tree, block)
end
private
def iterate(node, block)
block.call node
node.children.each { |n|
iterate(n, block)
}
end
end

Node
NodeNode
NodeNode
Top Down
Node
NodeNode
NodeNode

Node
class Node < Struct.new(:children)
def initialize(children = [])
super
end
def each(&block)
return enum_for(:each) unless block_given?
DepthFirst.new(self).each(&block)
end
end

Parallel Iteration
iter = tree1.each
tree2.all? { |node| node == iter.next }

To Inﬁnity, and
Beyond!

Inﬁnite Series
n → ∞

class Infinity
include Enumerable
def each
return enum_for(:each) unless block_given?
i = 0
loop do
yield i
i += 1
end
end
end

infinity = Enumerator.new do |yielder|
i = 0
loop do
yielder << i
i += 1
end
end

π
Pi

Mādhavan of Sangamagrāmam

pi = Enumerator.new do |yielder|
root_12 = Math.sqrt(12)
pi = 0
infinity.each do |k|
pi += (-1 / 3.0) ** k / (2 * k + 1).to_f
yielder << (root_12 * pi)
end
end

Generators

Iterator.is_a?
Generator

Generator Client

Generator Client
Value

Fibers

Abstract Methods

method = Proc.new { ... }
method.call # => "hello"
method.call # => "world"
method.call # => "how are you?"

method = Proc.new { ... }
method.call('1') # => "1: hello"
method.call('2') # => "2: world"
method.call('3') # => "3: how are you?"

Proc => Fiber

call => resume

method = Fiber.new { ... }
method.resume('1') # => "1: hello"
method.resume('2') # => "2: world"
method.resume('3') # => "3: how are you?"

Arbitrary Exits

method = Fiber.new do
Fiber.yield "hello"
Fiber.yield "world"
end
method.resume # => "hello"
method.resume # => "world"
method.resume # => nil
method.resume # => (FiberError)

Accept Parameters

method = Fiber.new do |val|
val = Fiber.yield "#{val}: hello"
val = Fiber.yield "#{val}: world"
val
end
method.resume(1) # => "1: hello"
method.resume(2) # => "2: world"
method.resume(3) # => 3
method.resume(4) # => (FiberError)

First "resume" is
the block argument

The rest are
returned by
Fiber.yield

Cool story bro

Callbacks -> FSM

YAML
because XML is boring

YAML
yes, I wrote nokogiri. :-P

Psych
Ruby 1.9.2 Only

class Foo < Psych::Handler
def start_document(...)
...
end
def end_document(...)
...
end
end
SAX Style Handler

Event Handler
class EventHandler
def initialize(fiber)
@fiber = fiber
end
def method_missing(name, *args)
@fiber.resume(name, args)
end
end

fiber = Fiber.new do |method, args|
stack = []
loop do
case method
when :start_mapping
stack << []
when :end_mapping
h = Hash[*stack.pop]
if stack.empty?
p h
else
stack.last << h
end
when :scalar
stack.last << args.first
end
method, args = Fiber.yield
end
end

case method
when :start_mapping
stack << []
when :end_mapping
h = Hash[*stack.pop]
if stack.empty?
stdout.resume h
else
stack.last << h
end
when :scalar
stack.last << args.first
end
method, args = Fiber.yield

StdOut Fiber
stdout = Fiber.new do |obj|
loop do
p obj
obj = Fiber.yield
end
end

Tied Together
parser = Psych::Parser.new EventHandler.new
fiber
parser.parse(<<-eoyaml)
---
- foo: { fiz: buzz }
- bar: { fiz: buzz }
- baz: { fiz: buzz }
eoyaml

Filtering Results

Foo Filter
foo_filter = Fiber.new do |hash|
loop do
stdout.resume hash if hash.key? 'foo'
hash = Fiber.yield
end
end

Output
{"foo"=>{"fiz"=>"buzz"}}

Scheduling
Threads Without Threads

P1
P2
Processes

P1
P2
Processes
fopen()
fread()
fseek()
fopen()
fopen()

Simple Example

Producer
producer = Fiber.new do
loop do
5.times do |i|
queue << "hello :#{i}"
end
Fiber.yield
end
end

Consumer
hungry = Fiber.new do
loop do
while !queue.empty?
puts queue.pop
end
Fiber.yield
end
end

Scheduler
fibers = [producer, hungry]
loop do
fibers.each do |fiber|
fiber.resume
end
end

Slow Consumer
slow = Fiber.new do
loop do
while !queue.empty?
sleep 1
puts "########### slow: " + queue.pop
Fiber.yield
end
Fiber.yield
end
end

Random Scheduler
fibers = [producer, hungry, slow]
loop do
fibers.sort_by { rand }.each do |fiber|
fiber.resume
end
end

"Processes"
•Anonymous
•We control when to "give up"

Homework
•Favor fast processes
•Favor queue ﬁllers
•Favor queue drainers
•Deal with multiple queues

Generators
The Fiber Edition

infinity = Fiber.new do
i = 0
loop do
Fiber.yield i
i += 1
end
end

Our Own
Enumerator

class SimpleEnum
def initialize &block
@fiber = Fiber.new do
block.call(self)
end
end
def << value
Fiber.yield value
end
def next
@fiber.resume
end
end

infinity = SimpleEnum.new do |yielder|
i = 0
loop do
yielder << i
i += 1
end
end

•Capture return value
•Add StopIteration Exception
Homework

Coroutines

Fiber implements
Coroutine

Coroutine Client
Value

Pipelines

Coroutine CoroutineGenerator

Our Friend, Inﬁnity
infinity = Fiber.new do
i = 0
loop do
Fiber.yield i
i += 1
end
end

Even Numbers
evens = Fiber.new do
loop do
value = infinity.resume
Fiber.yield value if value % 2 == 0
end
end

Three's
threes = Fiber.new do
loop do
value = evens.resume
Fiber.yield value if value % 3 == 0
end
end

Abstraction

Common Elements
•Source
•Sink
•Runloop
•Work

class Pipe < Struct.new(:source)
def initialize(source = nil)
super
@fiber = Fiber.new { self.loop }
end
def loop
while value = read
process value
end
end
def read; source.resume end
def write(value); Fiber.yield(value) end
def process(value); write(value) end
def resume; @fiber.resume end
end

class Infinity < Pipe
def loop
i = 0
while true
write i
i += 1
end
end
end
Inﬁnity (again)

Evens
class Evens < Pipe
def process(value)
super if value % 2 == 0
end
end

StdOut
class StdOut < Pipe
def write value
puts value
end
end

Connection
infinity = Infinity.new
even = Evens.new infinity
stdout = StdOut.new even
stdout.resume

Pretty Syntax
class Pipe
def |(other)
other.source = self
other
end
end

Connection
(Infinity.new | Evens.new | StdOut.new).resume

Producer / Consumer

Compiled Template
template = ERB.new('<%= "hello" %>')
puts template.src

Compiled Template
#coding:US-ASCII
_erbout = '';
_erbout.concat(( "hello" ).to_s);
_erbout.force_encoding(__ENCODING__)

Redirect Output

Compiled Template
erb = ERB.new('<%= "hello" %>', nil, nil, 'self')
puts erb.src

Compiled Template
#coding:US-ASCII
self = '';
self.concat(( "hello" ).to_s);
self.force_encoding(__ENCODING__)

Monkey Patch!
class ERB::Compiler
def pre_cmd= x; end
end
erb = ERB.new('<%= "hello" %>', nil, nil,
'self')
puts erb.src

Compiled Source
#coding:US-ASCII
self.concat(( "hello" ).to_s);
self.force_encoding(__ENCODING__)

ERb Pipe

ERb Pipe
class PipERB < Pipe
def initialize data
super()
@erb = ERB.new(data, nil, nil, 'self')
end
alias :concat :process
def read; @erb.result(binding) end
def force_encoding encoding; end
end

Pipe to StdOut
(PipERB.new("<%= 'hello' %>") | StdOut.new).resume

ERb Pipe + Rack

Slow Template
<ul>
<% a = b = 1 %>
<% 10.times do %>
<li><%= a %></li>
<% sleep(1) # simulate work %>
<% a, b = b, a + b %>
<% end %>
</ul>

Normal Handler
class ERBHandler
def initialize
@fib_template = DATA.read
end
def call env
result = [
ERB.new(@fib_template).result(binding)
]
[ 200, {'X-Hello' => 'World'}, result ]
end
end

Rack API
Demands "each"

Each Pipe
class Each < Pipe
def each(&block)
@block = block
resume
end
def process(value)
@block.call(value)
end
end

Each Pipe
(PipERB.new(DATA.read) | Each.new).each do |chunk|
print chunk
end
__END__
<ul>
<% a = b = 1 %>
<% 10.times do %>
<li><%= a %></li>
<% sleep 1; a, b = b, a + b %>
<% end %>
</ul>

Fiber Handler
class ERBHandler
def initialize
@fib_template = DATA.read
end
def call env
[ 200, {},
(PipERB.new(@fib_template) | Each.new)
]
end
end

Cooperative
Scheduling

ERb
Rack

Caveats?

Conclusion

Two Pipelines

Real Life?

Fibers are
Interesting!

Concurrency is
Interesting

But it requires a
shift in thinking

Install Ruby 1.9.2

Play with this at
home!

Questions?

RubyConf Brazil 2010

More Related Content

Similar to RubyConf Brazil 2010

Recently uploaded

RubyConf Brazil 2010