1. Python and MySQL
Ted Leung
Sun Microsystems

4. Python-DB
 In the beginning there was the DB-API
 PEP-249

5. MySQLdb
 MySQLdb written in Python
 _mysql written in C and is MySQL specific

6. import MySQLdb
conn = MySQLdb.connect(
host = 'localhost',
user = 'twl', Text
Text
Text
passwd='pass', Text
db='flickrmgr'
)

7. c = conn.cursor()
c.execute(quot;quot;quot;SELECT * from curator_photoquot;quot;quot;)
# read one row
pprint(c.fetchone())
# read all the rows
pprint(c.fetchall())
((3L,
3422686825L,
quot;OPG Toymaker's Doll 2009quot;,
2L,
'http://farm4.static.flickr.com/
3377/3422686825_fd57ea30e7.jpg',
datetime.datetime(2009, 4, 8, 1, 21, 44)),
(4L,
3422685683L,
quot;OPG Toymaker's Doll 2009quot;,
2L,
'http://farm4.static.flickr.com/
3608/3422685683_4d16829c19.jpg',
datetime.datetime(2009, 4, 8, 1, 20, 54)),

8. dc = conn.cursor(MySQLdb.cursors.DictCursor)
dc.execute(quot;quot;quot;SELECT * from curator_photoquot;quot;quot;)
pprint(dc.fetchmany(5))
({'flickr_id': 2147483647L,
'id': 2L,
'pub_date': datetime.datetime(2009, 4, 8, 1, 21, 44),
'title': quot;OPG Toymaker's Doll 2009quot;,
'url': 'http://farm4.static.flickr.com/
3377/3422686825_fd57ea30e7.jpg',
'user_id': 2L},
{'flickr_id': 3422686825L,
'id': 3L,
'pub_date': datetime.datetime(2009, 4, 8, 1, 21, 44),
'title': quot;OPG Toymaker's Doll 2009quot;,
'url': 'http://farm4.static.flickr.com/
3377/3422686825_fd57ea30e7.jpg',
'user_id': 2L})

9. c.execute(
quot;quot;quot;SELECT * from curator_photo WHERE id < %squot;quot;quot;,
(50,))
pprint(c.fetchall())

10. c.execute(
quot;quot;quot;INSERT INTO curator_user (flickr_id, name) VALUES
(%s, %s)quot;quot;quot;,
(quot;000quot;, quot;No Userquot;)
)
conn.commit()

11. Django
 Leading web application framework
 Lots of people coming to Python via Django today
 Model-View-Controller
 Command line management scripts
 It’s own Rails-like ORM
 Based on the Active Record pattern

12. Active Record Pattern
 Database table is wrapped in a class
 Each class instance is a row in the table
 Relationships expressd as foreign key constraints

13. DATABASE_ENGINE = 'mysql'
DATABASE_NAME = 'flickrmgr'
DATABASE_USER = 'twl'
DATABASE_PASSWORD = 'pass'

14. from django.db import models
class User(models.Model):
flickr_id = models.TextField()
name = models.TextField()
def __unicode__(self):
return self.name
class Group(models.Model):
flickr_id = models.TextField()
name = models.TextField()
throttle_count = models.IntegerField()
throttle_mode = models.TextField()
throttle_remaining = models.IntegerField()
def __unicode__(self):
return self.name

15. class Photo(models.Model):
flickr_id = models.IntegerField()
title = models.TextField()
user = models.ForeignKey(User)
groups = models.ManyToManyField(Group)
url = models.URLField()
pub_date = models.DateTimeField()
def __unicode__(self):
return self.title

16. from curator.models import Photo
from datetime import datetime
Photo.objects.all()
Photo.objects.filter(title__contains='PyCon')
Photo.objects.filter(title__contains='PyCon').exclude(
pub_date__lte=datetime(2009,4,1))
Photo.objects.filter(title__contains='PyCon').exclude(
pub_date__lte=datetime(2009,4,1))[2:4]

17. from django.contrib import admin
class PhotoAdmin(admin.ModelAdmin):
pass
class GroupAdmin(admin.ModelAdmin):
pass
class UserAdmin(admin.ModelAdmin):
pass
admin.site.register(Group, GroupAdmin)
admin.site.register(Photo, PhotoAdmin)
admin.site.register(User, UserAdmin)

18. def load_from_flickr(request):
api = API()
# 51035696189@N01
twl, created = User.objects.get_or_create(
flickr_id = '51035696189@N01',
name = 'Ted Leung'
)
if created:
twl.save()
photos = api.list_user_info(
'http://www.flickr.com/photos/twleung')

19. for photo in photos:
flickr_id, title, pub_date, url, pools = photo
new_photo = Photo()
new_photo.flickr_id = flickr_id
new_photo.title = title
new_photo.user = twl
new_photo.pub_date =
datetime.fromtimestamp(int(pub_date))
new_photo.url = url
new_photo.save()

20. # do pools
for pool in pools:
new_pool, created=Group.objects.get_or_create(
flickr_id = pool[0],
name = pool[1],
throttle_count = pool[2],
throttle_mode = pool[3],
throttle_remaining = pool[4]
)
new_photo.groups.add(new_pool)
if created:
new_pool.save()

21. output = '''
<html>
<head>
<title>Bulk loading from Flickr</title>
</head>
<body>
</body>
</html>
'''
return HttpResponse(output)

22. from django.conf.urls.defaults import *
from curator.models import User, Group, Photo
from curator.views import load_from_flickr
photo_info_dict = {
'queryset': Photo.objects.all(),
'date_field': 'pub_date',
}
urlpatterns = patterns(
'',
(r'^$',
'django.views.generic.date_based.archive_index',
photo_info_dict),
(r'^load/$',
load_from_flickr),
)

23. Transactions
 Commit on save or delete
 Commit at HTTP request/response boundaries
 commit_manually decorator

24. Connection Pooling
 Important as systems scale up
 No framework wide solution yet

25. Migration
 South
 Integrates with manage.py
 Can automatically migrate models

26. ../bin/django startmigration curator --initial
Creating migrations directory at '/Users/twl/work/
mysql-2009/django/flickrmgr/curator/migrations'...
Creating __init__.py in '/Users/twl/work/mysql-2009/
django/flickrmgr/curator/migrations'...
+ Added model 'curator.Photo'
+ Added model 'curator.Group'
+ Added model 'curator.User'
+ Added field 'curator.Photo.groups'
Created 0001_initial.py.

27. class Photo(models.Model):
flickr_id = models.IntegerField()
title = models.TextField()
user = models.ForeignKey(User)
groups = models.ManyToManyField(Group)
url = models.URLField()
pub_date = models.DateTimeField()
visible = models.BooleanField()

28. ../bin/django startmigration curator add_visible --auto
+ Added field 'curator.photo.visible'
Created 0002_add_visible.py.

29. from south.db import db
from django.db import models
from curator.models import *
class Migration:
def forwards(self, orm):
# Adding field 'Photo.visible'
db.add_column('curator_photo', 'visible',
models.BooleanField())
def backwards(self, orm):
# Deleting field 'Photo.visible'
db.delete_column('curator_photo', 'visible')

30. SQLObject
 ORM using Active Record pattern
 Tight coupling to Python classes
 Used in TurboGears 1

31. connection_string='mysql://twl:pass@localhost/sqlobject'
connection=connectionForURI(connection_string)
sqlhub.processConnection = connection
try:
User.createTable(ifNotExists=True)
Photo.createTable(ifNotExists=True)
FlickrGroup.createTable(ifNotExists=True)
except dberrors.OperationalError, oe:
print oe

32. class User(SQLObject):
flickr_id = StringCol()
name = StringCol()
class Photo(SQLObject):
flickr_id = StringCol()
title = StringCol()
user = ForeignKey('User')
groups = RelatedJoin('FlickrGroup')
url = StringCol()
pub_date = DateTimeCol()
class FlickrGroup(SQLObject):
flickr_id = StringCol()
name = StringCol()
throttle_count = IntCol()
throttle_mode = StringCol()
throttle_remaining = IntCol()
photos = RelatedJoin('Photo')

33. pycon_photos =
list(Photo.select(Photo.q.title.contains('PyCon')))
pprint(pycon_photos)
pycon_apr_photos =
list(Photo.select(AND(Photo.q.title.contains('PyCon'),
Photo.q.pub_date > datetime(2009,4,1)
)))
pprint(pycon_apr_photos)

34. api = API()
twl = User(
flickr_id = '51035696189@N01',
name = 'Ted Leung'
)
photos = api.list_user_info(
'http://www.flickr.com/photos/twleung')

35. for photo in photos:
print photo
flickr_id, title, pub_date, url, pools = photo
new_photo = Photo(
flickr_id = flickr_id,
title = title,
user = twl,
pub_date = datetime.fromtimestamp(int(pub_date)),
url = url
)

36. # do pools
for pool in pools:
new_pool =
list(FlickrGroup.select(
FlickrGroup.q.flickr_id == pool[0]))
if len(new_pool) > 0:
new_pool = new_pool[0]
else:
new_pool = None
if not new_pool:
new_pool = FlickrGroup(
flickr_id = pool[0],
name = pool[1],
throttle_count = pool[2],
throttle_mode = pool[3],
throttle_remaining = pool[4]
)
new_photo.addFlickrGroup(new_pool)

37. Transactions
txn = connection.transaction()
p = Photo.get(1, txn)
p.title = ‘updated photo’
txn.commit()

38. Connection Pooling
 Connection pooling is built in and on by default

39. sqlmeta
class Photo(SQLObject):
class sqlmeta:
lazyUpdate = True
cacheValues = False
flickr_id = StringCol()
title = StringCol()
user = ForeignKey('User')
groups = RelatedJoin('FlickrGroup')
url = StringCol()
pub_date = DateTimeCol()

40. Events
from sqlobject.events import listen
from sqlobject.events RowUpdateSignal, RowCreatedSignal
def update_listener(instance, kwargs):
kwargs['pub_date'] = datetime.datetime.now()
def created_listener(kwargs, post_funcs):
print “created photo %s” % (kwargs[‘title’])
listen(update_listener, Photo, RowUpdateSignal)
listen(created_listener, Photo, RowCreatedSignal)

41. SQLAlchemy
 Python’s answer to Hibernate
 Low level SQL manipulations
 High Level ORM
 Used in TurboGears 2

42. Low Level SQL

43. from sqlalchemy import create_engine
from sqlalchemy import Table, Column, Integer, String,
Text, DateTime, MetaData, ForeignKey
from sqlalchemy import select
engine = create_engine(
'mysql://twl:pass@localhost/sqlobject',echo=True)
metadata = MetaData()

44. users_table = Table('user', metadata,
Column('id', Integer, primary_key=True),
Column('flickr_id', Text),
Column('name', Text)
)
groups_table = Table('group', metadata,
Column('id', Integer, primary_key=True),
Column('flickr_id', Text),
Column('name', Text),
Column('throttle_count', Integer),
Column('throttle_mode', Text),
Column('throttle_remaining', Integer)
)
photos_table = Table('photo', metadata,
Column('id', Integer, primary_key=True),
Column('flickr_id', Text),
Column('title', Text),
Column('user_id', Integer,
ForeignKey(‘user.id’)),
Column('url', Text),
Column('pub_date', DateTime)
)

45. metadata.create_all(engine)
conn = engine.connect()
s = select([users_table])
result = conn.execute(s)
for row in result:
print row
s = select([photos_table],
photos_table.c.title.contains('PyCon'))
for row in conn.execute(s):
print row

46. ins = users.insert().values(name='Thomas Hawk',
flickr_id='000')
result = conn.execute(ins)
update =
users.update().where(
users.c.name=='Thomas Hawk'
).values(
flickr_id='51035555243@N01'))
result = conn.execute(update)

47. Transactions
 Regular manual control
 Autocommit - per session
 2PC

48. Connection Pooling
 Built-in framework
 Done at engine configuration time

49. Other Low Level SQL features
 Unions and other Set operations
 Scalar Selects
 Correlated Subqueries
 Ordering, Grouping, Offsetting
 Correlated Updates

50. ORM
 Mapping
 Declarative

51. class User(object):
def __repr__(self):
return quot;<User('%s','%s','%s')>quot; % (
self.id, self.flickr_id, self.name)
class Group(object):
def __repr__(self):
return quot;<Group('%s','%s','%s','%s')>quot; % (
self.id,
self.flickr_id,
self.name,
self.throttle_remaining
)
class Photo(object):
def __repr__(self):
return quot;<Photo('%s','%s','%s','%s')>quot; % (
self.id,
self.flickr_id,
self.title,
self.url
)

52. metadata = MetaData()
flickr_group_photo = Table(
'flickr_group_photo', metadata,
Column('flickr_group_id', Integer,
ForeignKey('flickr_group.id')),
Column('photo_id', Integer,
ForeignKey('photo.id')))

53. from sqlalchemy.orm import mapper, sessionmaker
engine = create_engine(
'mysql://twl:pass@localhost/sqlobject')

54. Session = sessionmaker(bind = engine)
session = Session()
mapper(User, users_table, properties = {
'photos' : relation(Photo, backref='user')
})
mapper(Group, groups_table, properties = {
'photos' : relation(Photo,
secondary=flickr_group_photo,
backref='groups')
})
mapper(Photo, photos_table)
for u in session.query(User):
print u
for g in session.query(Group):
print g
for p in session.query(Photo):
print p

55. Declarative ORM

56. from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
from sqlalchemy import Column, ForeignKey
from sqlalchemy import Integer, String, Text, DateTime
Base = declarative_base()
class User(Base):
__tablename__ = 'user'
id = Column(Integer, primary_key=True)
flickr_id = Column(Text)
name = Column(Text)
def __repr__(self):
return quot;<User('%s','%s','%s')>quot; % (
self.id, self.flickr_id, self.name)

57. class Group(Base):
__tablename__ = 'flickr_group'
id = Column(Integer, primary_key=True)
flickr_id = Column(Text)
name = Column(Text)
throttle_count = Column(Integer)
throttle_mode = Column(Text)
throttle_remaining = Column(Integer)
def __repr__(self):
return quot;<Group('%s','%s','%s','%s')>quot; % (
self.id,
self.flickr_id,
self.name,
self.throttle_remaining
)

58. class Photo(Base):
__tablename__ = 'photo'
id = Column(Integer, primary_key=True)
flickr_id = Column(Text)
title = Column(Text)
user_id = Column(Integer,ForeignKey('user.id'))
user = relation(User,
backref=backref('photos', order_by=id))
groups = relation('Group',
secondary=flickr_group_photo,
backref=backref('photos'))
url = Column(Text)
pub_date = Column(DateTime)
def __repr__(self):
return quot;<Photo('%s','%s','%s','%s')>quot; % (
self.id,
self.flickr_id,
self.title,
self.url
)

59. Session = sessionmaker(bind = engine)
session = Session()
for u in session.query(User):
print u
for g in session.query(Group):
print g
for p in session.query(Photo):
print p

60. And more
 Advanced mapping
 Multiple table mapping

61. Elixir
 a simpler way of doing declarative ORM for
SQLAlchemy

62. from elixir import *
class User(Entity):
using_options(tablename = 'user')
flickr_id = Field(Text)
name = Field(Text)
photos = OneToMany('Photo')
def __repr__(self):
return quot;<User('%s','%s','%s')>quot; % (
self.id, self.flickr_id, self.name)

63. class Group(Entity):
using_options(tablename = 'flickr_group')
flickr_id = Field(Text)
name = Field(Text)
throttle_count = Field(Integer)
throttle_mode = Field(Text)
throttle_remaining = Field(Integer)
def __repr__(self):
return quot;<Group('%s','%s','%s','%s')>quot; % (
self.id,
self.flickr_id,
self.name,
self.throttle_remaining
)

64. class Photo(Entity):
using_options(tablename = 'photo')
flickr_id = Field(Text)
title = Field(Text)
url = Column(Text)
pub_date = Column(DateTime)
groups = ManyToMany('Group')
user = ManyToOne('User')
def __repr__(self):
return quot;<Photo('%s','%s','%s','%s')>quot; % (
self.id,
self.flickr_id,
self.title,
self.url
)

65. engine = create_engine(
'mysql://twl:pass@localhost/sqlobject', echo=True)
Session = sessionmaker(bind = engine)
session = Session()
metadata = MetaData()
metadata.bind='mysql://twl:pass@localhost/sqlobject'
metadata.bind.echo = True
setup_all()
for u in session.query(User):
print u
for g in session.query(Group):
print g
for p in session.query(Photo):
print print

66. SqlSoup
from sqlalchemy.ext.sqlsoup import SqlSoup
soup = SqlSoup('mysql://twl:pass@localhost/
sqlobject')
soup.photo.all()

67. soup.photo.order_by(soup.photo.title).all()
[MappedPhoto(id=71L,flickr_id='3286255071',title='',
user_id=1L,url='http://farm4.static.flickr.com/
3249/3286255071_ff168f220b.jpg',pub_date=datetime.da
tetime(2009, 2, 16, 20, 47, 56)),
MappedPhoto(id=72L,flickr_id='3287070244',title='',u
ser_id=1L,url='http://farm4.static.flickr.com/
3298/3287070244_87a2a1b3ed.jpg',pub_date=datetime.da
tetime(2009, 2, 16, 20, 47, 22)),
MappedPhoto(id=46L,flickr_id='3395647634',title='And
y Dustman',user_id=1L,url='http://
farm4.static.flickr.com/
3554/3395647634_cc0c9f5a0a.jpg',pub_date=datetime.da
tetime(2009, 3, 29, 9, 7, 34)),
MappedPhoto(id=73L,flickr_id='3277628077',title='Bai
nbridge Island Chinese New Year
2009',user_id=1L,url='http://
farm4.static.flickr.com/
3334/3277628077_78025002f5.jpg',pub_date=datetime.da
tetime(2009, 2, 13, 23, 31, 52)),

68. Migrations
 sqlalchemy-migrate
 Version control a database
 A repository of upgrade scripts

69. from sqlalchemy import *
from migrate import *
metadata = MetaData(migrate_engine)
photos_table = Table('photo', metadata,
Column('id', Integer, primary_key=True),
Column('flickr_id', Text),
Column('title', Text),
Column('user_id', Integer),
Column('url', Text),
Column('pub_date', DateTime),
)
visible_col = Column('visible', Integer)
def upgrade():
visible_col.create(photos_table)
assert visible_col is photos_table.c.visible
def downgrade():
visible_col.drop()

70. from sqlalchemy import *
from migrate import *
metadata = MetaData(migrate_engine)
photos_table = Table('photo', metadata,
Column('id', Integer, primary_key=True),
Column('flickr_id', Text),
Column('title', Text),
Column('user_id', Integer),
Column('url', Text),
Column('pub_date', DateTime),
Column('visible', Integer)
)
col = photos_table.c.visible
def upgrade():
col.alter(type=Boolean, nullable=False)
def downgrade():
col.alter(type=Integer, nullable=True)

71. Migration operations
 Create/Drop a table
 Create/Drop/Alter a column
 Create/Drop index
 Create/Drop primary/foreign key constraints

72. Summary
 Django for Django
 SQLAlchemy for the rest

73. Thanks!
 ted.leung@sun.com
 twitter: twleung