A talk on Python coroutines for asynchronous programming, both *now* in Tornado and Toro, and in the future in Tulip and Python 3.4.

1. Python Coroutines
Present and Future
A. Jesse Jiryu Davis
http://emptysquare.net
10gen

2. Python Coroutines
Present and Future
Agenda:
• Coroutines are wonderful for async
• …but weird.
• Understanding the weirdness.
• Coroutine Kung Fu.
• The Future!

3. Coroutines are
wonderful for async

4. Async with callback
from tornado.web import RequestHandler
class AsyncHandler(RequestHandler):
@asynchronous
def get(self):
http_client = AsyncHTTPClient()
http_client.fetch(
"http://example.com",
callback=self.on_fetch)
def on_fetch(self, response):
do_something_with_response(response)
self.render("template.html")

5. Async with coroutine
from tornado.web import RequestHandler
class GenAsyncHandler(RequestHandler):
@asynchronous
@gen.coroutine
def get(self):
http_client = AsyncHTTPClient()
response = yield http_client.fetch(
"http://example.com")
do_something_with_response(response)
self.render("template.html")

6. Async with coroutine
from tornado.web import RequestHandler
class GenAsyncHandler(RequestHandler):
@asynchronous
@gen.coroutine
def get(self):
http_client = AsyncHTTPClient()
future = http_client.fetch(
"http://example.com")
response = yield future
do_something_with_response(response)
self.render("template.html")

7. Agenda:
• Coroutines are wonderful for Async I/O
• …but weird.
• yield
• future
• coroutine

8. yield
def make_generator():
yield 0
yield 1
g = make_generator()
print(g)
# <generator object make_generator
at 0x1006b4b40>

9. def make_generator():
yield 0
yield 1
g = make_generator()
for i in g:
print(i)

10. def make_generator():
yield 0
yield 1
g = make_generator()
while True:
try:
i = g.__next__()
print(i)
except StopIteration:
break

11. def make_generator():
yield 0
yield 1
g = make_generator()
while True:
try:
i = g.send(None)
print(i)
except StopIteration:
break

12. Prints:
def make_generator():
return_value = yield 0
0
print 'got', return_value got 10
return_value = yield 1 1
print 'got', return_value got 11
g = make_generator()
i = g.send(None) # Start g
while True:
try:
print(i)
i = g.send(i + 10)
except StopIteration:
break

13. Agenda:
• Coroutines are wonderful for Async I/O
• …but weird.
• yield
• future
• coroutine

14. Future
class GenAsyncHandler(RequestHandler):
@asynchronous
@gen.coroutine
def get(self):
http_client = AsyncHTTPClient()
future = http_client.fetch(
"http://example.com")
response = yield future
do_something_with_response(response)
self.render("template.html")

15. future = Future()
future.done() # False
future.set_result('foo')
future.set_exception(
SomeException('error message'))
future.add_done_callback(callback)
# Return result or raise error
future.result()

16. class GenAsyncHandler(RequestHandler):
@asynchronous
@gen.coroutine
def get(self):
http_client = AsyncHTTPClient()
future = http_client.fetch(
"http://example.com")
response = yield future
do_something_with_response(response)
self.render("template.html")

17. Agenda:
• Coroutines are wonderful for Async I/O
• …but weird.
• yield
• future
• coroutine

18. # part of the Tornado framework
class Runner(object):
def __init__(self, make_generator):
self.gen = make_generator()
# Starts done, with result None
self.future = NullFuture()

19. class Runner(object):
# ... "recurse"
def run(self):
while True:
if not self.future.done():
self.future.add_done_callback(self.run)
return
value = self.future.result()
try:
self.future = self.gen.send(value)
except (StopIteration, Return) as e:
return

20. class GenAsyncHandler(RequestHandler):
@asynchronous
@gen.coroutine
def get(self):
http_client = AsyncHTTPClient()
future = http_client.fetch(
"http://example.com")
response = yield future
do_something_with_response(response)
self.render("template.html")

21. Agenda:
• Coroutines are wonderful for Async I/O
• …but weird.
• Handling the weirdness.

22. @gen.coroutine
def print_code():
response = yield get('http://example.com')
print response.code
@gen.coroutine weird
def get(url):
client = AsyncHTTPClient()
response = yield client.fetch(url)
raise gen.Return(response)
print_code()

23. @gen.coroutine
def print_code():
future = get('http://example.com')
response = yield future
print response.code
@gen.coroutine
def get(url):
client = AsyncHTTPClient()
response = yield client.fetch(url)
raise gen.Return(response)
print_code()
weird

24. Python 3.3
@gen.coroutine
def print_code():
future = get('http://example.com')
response = yield future
print response.code
@gen.coroutine
def get(url):
client = AsyncHTTPClient()
response = yield client.fetch(url)
return response
print_code()
normal

25. @gen.coroutine
def print_code():
try:
code = yield get('http://example.com')
print code
except HTTPError, e: normal
print e
@gen.coroutine
def get(url):
client = AsyncHTTPClient()
response = yield client.fetch(url)
raise gen.Return(response.code)
print_code()

26. Agenda:
• Coroutines are wonderful for Async I/O
• …but weird.
• Handling the weirdness.
• Coroutine Kung Fu.

27. Fan-out
@gen.coroutine
def f():
client = AsyncHTTPClient()
responses = yield [
client.fetch('http://mongodb.org'),
client.fetch('http://10gen.com')]
print responses
f()

28. Fan-out
@gen.coroutine
def f():
client = AsyncHTTPClient()
future0 = client.fetch('http://mongodb.org')
future1 = client.fetch('http://10gen.com')
responses = yield [future0, future1]
print responses
f()

29. Toro
Synchronization Primitives for
Tornado Coroutines

30. event = toro.Event()
@gen.coroutine
def waiter():
print "I'll wait right here"
yield event.wait() # Yield a Future
print "I'm done waiting"
@gen.coroutine
def setter():
print "About to set"
event.set()
print "Done setting"
waiter()
setter()

31. q = toro.Queue(maxsize=3)
@gen.coroutine
def producer():
for item in range(5):
print 'Sending', item
yield q.put(item)
@gen.coroutine
def consumer():
while True:
item = yield q.get()
print 'tt', 'Got', item
consumer()
producer()

32. $ python producer_consumer.py
Sending 0
! ! Got 0
Sending 1
! ! Got 1
Sending 2
! ! Got 2
Sending 3
! ! Got 3
Sending 4
! ! Got 4

33. Agenda:
• Coroutines are wonderful for Async I/O
• …but weird.
• Handling the weirdness.
• Kung Fu.
• The Future!

34. Tulip
• A standard event loop
• A standard coroutine library
• For inclusion in Python 3.4

35. yield from

36. @gen.coroutine
def print_code(): Tornado
response = yield get('http://example.com')
print response.code
@gen.coroutine
def get(url):
client = AsyncHTTPClient()
response = yield client.fetch(url)
raise gen.Return(response)
@tulip.coroutine
def print_code():
Tulip
response = yield from get('http://example.com')
print(response.status)
@tulip.coroutine
"yield from"
def get(url):
request = tulip.http.request('GET', url)
response = yield from request
return response
normal return

37. Guido
on
"The Difference Between
yield and yield-from":
http://bit.ly/yieldfrom

38. q = tulip.queues.Queue(maxsize=3)
@tulip.coroutine
def producer():
for item in range(5):
print('Sending', item)
yield from q.put(item)
@tulip.coroutine
def consumer():
while True:
item = yield from q.get()
print('tt', 'Got', item)
Task(consumer())
Task(producer())

39. StopIteration
A. Jesse Jiryu Davis
http://emptysquare.net
10gen