MySQL User Conference 2009: Python and MySQL

Python and MySQL Ted Leung Sun Microsystems

Python-DB  In the beginning there was the DB-API  PEP-249

MySQLdb  MySQLdb written in Python  _mysql written in C and is MySQL specific

import MySQLdb conn = MySQLdb.connect( host = 'localhost', user = 'twl', Text Text Text passwd='pass', Text db='flickrmgr' )

c = conn.cursor() c.execute(\"\"\"SELECT * from curator_photo\"\"\") # read one row pprint(c.fetchone()) # read all the rows pprint(c.fetchall()) ((3L, 3422686825L, \"OPG Toymaker's Doll 2009\", 2L, 'http://farm4.static.flickr.com/ 3377/3422686825_fd57ea30e7.jpg', datetime.datetime(2009, 4, 8, 1, 21, 44)), (4L, 3422685683L, \"OPG Toymaker's Doll 2009\", 2L, 'http://farm4.static.flickr.com/ 3608/3422685683_4d16829c19.jpg', datetime.datetime(2009, 4, 8, 1, 20, 54)),

dc = conn.cursor(MySQLdb.cursors.DictCursor) dc.execute(\"\"\"SELECT * from curator_photo\"\"\") pprint(dc.fetchmany(5)) ({'flickr_id': 2147483647L, 'id': 2L, 'pub_date': datetime.datetime(2009, 4, 8, 1, 21, 44), 'title': \"OPG Toymaker's Doll 2009\", 'url': 'http://farm4.static.flickr.com/ 3377/3422686825_fd57ea30e7.jpg', 'user_id': 2L}, {'flickr_id': 3422686825L, 'id': 3L, 'pub_date': datetime.datetime(2009, 4, 8, 1, 21, 44), 'title': \"OPG Toymaker's Doll 2009\", 'url': 'http://farm4.static.flickr.com/ 3377/3422686825_fd57ea30e7.jpg', 'user_id': 2L})

c.execute( \"\"\"SELECT * from curator_photo WHERE id < %s\"\"\", (50,)) pprint(c.fetchall())

c.execute( \"\"\"INSERT INTO curator_user (flickr_id, name) VALUES (%s, %s)\"\"\", (\"000\", \"No User\") ) conn.commit()

Django  Leading web application framework  Lots of people coming to Python via Django today  Model-View-Controller  Command line management scripts  It’s own Rails-like ORM  Based on the Active Record pattern

Active Record Pattern  Database table is wrapped in a class  Each class instance is a row in the table  Relationships expressd as foreign key constraints

DATABASE_ENGINE = 'mysql' DATABASE_NAME = 'flickrmgr' DATABASE_USER = 'twl' DATABASE_PASSWORD = 'pass'

from django.db import models class User(models.Model): flickr_id = models.TextField() name = models.TextField() def __unicode__(self): return self.name class Group(models.Model): flickr_id = models.TextField() name = models.TextField() throttle_count = models.IntegerField() throttle_mode = models.TextField() throttle_remaining = models.IntegerField() def __unicode__(self): return self.name

class Photo(models.Model): flickr_id = models.IntegerField() title = models.TextField() user = models.ForeignKey(User) groups = models.ManyToManyField(Group) url = models.URLField() pub_date = models.DateTimeField() def __unicode__(self): return self.title

from curator.models import Photo from datetime import datetime Photo.objects.all() Photo.objects.filter(title__contains='PyCon') Photo.objects.filter(title__contains='PyCon').exclude( pub_date__lte=datetime(2009,4,1)) Photo.objects.filter(title__contains='PyCon').exclude( pub_date__lte=datetime(2009,4,1))[2:4]

from django.contrib import admin class PhotoAdmin(admin.ModelAdmin): pass class GroupAdmin(admin.ModelAdmin): pass class UserAdmin(admin.ModelAdmin): pass admin.site.register(Group, GroupAdmin) admin.site.register(Photo, PhotoAdmin) admin.site.register(User, UserAdmin)

def load_from_flickr(request): api = API() # 51035696189@N01 twl, created = User.objects.get_or_create( flickr_id = '51035696189@N01', name = 'Ted Leung' ) if created: twl.save() photos = api.list_user_info( 'http://www.flickr.com/photos/twleung')

for photo in photos: flickr_id, title, pub_date, url, pools = photo new_photo = Photo() new_photo.flickr_id = flickr_id new_photo.title = title new_photo.user = twl new_photo.pub_date = datetime.fromtimestamp(int(pub_date)) new_photo.url = url new_photo.save()

# do pools for pool in pools: new_pool, created=Group.objects.get_or_create( flickr_id = pool[0], name = pool[1], throttle_count = pool[2], throttle_mode = pool[3], throttle_remaining = pool[4] ) new_photo.groups.add(new_pool) if created: new_pool.save()

output = ''' <html> <head> <title>Bulk loading from Flickr</title> </head> <body> </body> </html> ''' return HttpResponse(output)

from django.conf.urls.defaults import * from curator.models import User, Group, Photo from curator.views import load_from_flickr photo_info_dict = { 'queryset': Photo.objects.all(), 'date_field': 'pub_date', } urlpatterns = patterns( '', (r'^$', 'django.views.generic.date_based.archive_index', photo_info_dict), (r'^load/$', load_from_flickr), )

Transactions  Commit on save or delete  Commit at HTTP request/response boundaries  commit_manually decorator

Connection Pooling  Important as systems scale up  No framework wide solution yet

Migration  South  Integrates with manage.py  Can automatically migrate models

../bin/django startmigration curator --initial Creating migrations directory at '/Users/twl/work/ mysql-2009/django/flickrmgr/curator/migrations'... Creating __init__.py in '/Users/twl/work/mysql-2009/ django/flickrmgr/curator/migrations'... + Added model 'curator.Photo' + Added model 'curator.Group' + Added model 'curator.User' + Added field 'curator.Photo.groups' Created 0001_initial.py.

class Photo(models.Model): flickr_id = models.IntegerField() title = models.TextField() user = models.ForeignKey(User) groups = models.ManyToManyField(Group) url = models.URLField() pub_date = models.DateTimeField() visible = models.BooleanField()

../bin/django startmigration curator add_visible --auto + Added field 'curator.photo.visible' Created 0002_add_visible.py.

from south.db import db from django.db import models from curator.models import * class Migration: def forwards(self, orm): # Adding field 'Photo.visible' db.add_column('curator_photo', 'visible', models.BooleanField()) def backwards(self, orm): # Deleting field 'Photo.visible' db.delete_column('curator_photo', 'visible')

SQLObject  ORM using Active Record pattern  Tight coupling to Python classes  Used in TurboGears 1

connection_string='mysql://twl:pass@localhost/sqlobject' connection=connectionForURI(connection_string) sqlhub.processConnection = connection try: User.createTable(ifNotExists=True) Photo.createTable(ifNotExists=True) FlickrGroup.createTable(ifNotExists=True) except dberrors.OperationalError, oe: print oe

class User(SQLObject): flickr_id = StringCol() name = StringCol() class Photo(SQLObject): flickr_id = StringCol() title = StringCol() user = ForeignKey('User') groups = RelatedJoin('FlickrGroup') url = StringCol() pub_date = DateTimeCol() class FlickrGroup(SQLObject): flickr_id = StringCol() name = StringCol() throttle_count = IntCol() throttle_mode = StringCol() throttle_remaining = IntCol() photos = RelatedJoin('Photo')

pycon_photos = list(Photo.select(Photo.q.title.contains('PyCon'))) pprint(pycon_photos) pycon_apr_photos = list(Photo.select(AND(Photo.q.title.contains('PyCon'), Photo.q.pub_date > datetime(2009,4,1) ))) pprint(pycon_apr_photos)

api = API() twl = User( flickr_id = '51035696189@N01', name = 'Ted Leung' ) photos = api.list_user_info( 'http://www.flickr.com/photos/twleung')

for photo in photos: print photo flickr_id, title, pub_date, url, pools = photo new_photo = Photo( flickr_id = flickr_id, title = title, user = twl, pub_date = datetime.fromtimestamp(int(pub_date)), url = url )

# do pools for pool in pools: new_pool = list(FlickrGroup.select( FlickrGroup.q.flickr_id == pool[0])) if len(new_pool) > 0: new_pool = new_pool[0] else: new_pool = None if not new_pool: new_pool = FlickrGroup( flickr_id = pool[0], name = pool[1], throttle_count = pool[2], throttle_mode = pool[3], throttle_remaining = pool[4] ) new_photo.addFlickrGroup(new_pool)

Transactions txn = connection.transaction() p = Photo.get(1, txn) p.title = ‘updated photo’ txn.commit()

Connection Pooling  Connection pooling is built in and on by default

sqlmeta class Photo(SQLObject): class sqlmeta: lazyUpdate = True cacheValues = False flickr_id = StringCol() title = StringCol() user = ForeignKey('User') groups = RelatedJoin('FlickrGroup') url = StringCol() pub_date = DateTimeCol()

Events from sqlobject.events import listen from sqlobject.events RowUpdateSignal, RowCreatedSignal def update_listener(instance, kwargs): kwargs['pub_date'] = datetime.datetime.now() def created_listener(kwargs, post_funcs): print “created photo %s” % (kwargs[‘title’]) listen(update_listener, Photo, RowUpdateSignal) listen(created_listener, Photo, RowCreatedSignal)

SQLAlchemy  Python’s answer to Hibernate  Low level SQL manipulations  High Level ORM  Used in TurboGears 2

Low Level SQL

from sqlalchemy import create_engine from sqlalchemy import Table, Column, Integer, String, Text, DateTime, MetaData, ForeignKey from sqlalchemy import select engine = create_engine( 'mysql://twl:pass@localhost/sqlobject',echo=True) metadata = MetaData()

users_table = Table('user', metadata, Column('id', Integer, primary_key=True), Column('flickr_id', Text), Column('name', Text) ) groups_table = Table('group', metadata, Column('id', Integer, primary_key=True), Column('flickr_id', Text), Column('name', Text), Column('throttle_count', Integer), Column('throttle_mode', Text), Column('throttle_remaining', Integer) ) photos_table = Table('photo', metadata, Column('id', Integer, primary_key=True), Column('flickr_id', Text), Column('title', Text), Column('user_id', Integer, ForeignKey(‘user.id’)), Column('url', Text), Column('pub_date', DateTime) )

metadata.create_all(engine) conn = engine.connect() s = select([users_table]) result = conn.execute(s) for row in result: print row s = select([photos_table], photos_table.c.title.contains('PyCon')) for row in conn.execute(s): print row

ins = users.insert().values(name='Thomas Hawk', flickr_id='000') result = conn.execute(ins) update = users.update().where( users.c.name=='Thomas Hawk' ).values( flickr_id='51035555243@N01')) result = conn.execute(update)

Transactions  Regular manual control  Autocommit - per session  2PC

Connection Pooling  Built-in framework  Done at engine configuration time

Other Low Level SQL features  Unions and other Set operations  Scalar Selects  Correlated Subqueries  Ordering, Grouping, Offsetting  Correlated Updates

ORM  Mapping  Declarative

class User(object): def __repr__(self): return \"<User('%s','%s','%s')>\" % ( self.id, self.flickr_id, self.name) class Group(object): def __repr__(self): return \"<Group('%s','%s','%s','%s')>\" % ( self.id, self.flickr_id, self.name, self.throttle_remaining ) class Photo(object): def __repr__(self): return \"<Photo('%s','%s','%s','%s')>\" % ( self.id, self.flickr_id, self.title, self.url )

metadata = MetaData() flickr_group_photo = Table( 'flickr_group_photo', metadata, Column('flickr_group_id', Integer, ForeignKey('flickr_group.id')), Column('photo_id', Integer, ForeignKey('photo.id')))

from sqlalchemy.orm import mapper, sessionmaker engine = create_engine( 'mysql://twl:pass@localhost/sqlobject')

Session = sessionmaker(bind = engine) session = Session() mapper(User, users_table, properties = { 'photos' : relation(Photo, backref='user') }) mapper(Group, groups_table, properties = { 'photos' : relation(Photo, secondary=flickr_group_photo, backref='groups') }) mapper(Photo, photos_table) for u in session.query(User): print u for g in session.query(Group): print g for p in session.query(Photo): print p

Declarative ORM

from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import sessionmaker from sqlalchemy import Column, ForeignKey from sqlalchemy import Integer, String, Text, DateTime Base = declarative_base() class User(Base): __tablename__ = 'user' id = Column(Integer, primary_key=True) flickr_id = Column(Text) name = Column(Text) def __repr__(self): return \"<User('%s','%s','%s')>\" % ( self.id, self.flickr_id, self.name)

class Group(Base): __tablename__ = 'flickr_group' id = Column(Integer, primary_key=True) flickr_id = Column(Text) name = Column(Text) throttle_count = Column(Integer) throttle_mode = Column(Text) throttle_remaining = Column(Integer) def __repr__(self): return \"<Group('%s','%s','%s','%s')>\" % ( self.id, self.flickr_id, self.name, self.throttle_remaining )

class Photo(Base): __tablename__ = 'photo' id = Column(Integer, primary_key=True) flickr_id = Column(Text) title = Column(Text) user_id = Column(Integer,ForeignKey('user.id')) user = relation(User, backref=backref('photos', order_by=id)) groups = relation('Group', secondary=flickr_group_photo, backref=backref('photos')) url = Column(Text) pub_date = Column(DateTime) def __repr__(self): return \"<Photo('%s','%s','%s','%s')>\" % ( self.id, self.flickr_id, self.title, self.url )

Session = sessionmaker(bind = engine) session = Session() for u in session.query(User): print u for g in session.query(Group): print g for p in session.query(Photo): print p

And more  Advanced mapping  Multiple table mapping

Elixir  a simpler way of doing declarative ORM for SQLAlchemy

from elixir import * class User(Entity): using_options(tablename = 'user') flickr_id = Field(Text) name = Field(Text) photos = OneToMany('Photo') def __repr__(self): return \"<User('%s','%s','%s')>\" % ( self.id, self.flickr_id, self.name)

class Group(Entity): using_options(tablename = 'flickr_group') flickr_id = Field(Text) name = Field(Text) throttle_count = Field(Integer) throttle_mode = Field(Text) throttle_remaining = Field(Integer) def __repr__(self): return \"<Group('%s','%s','%s','%s')>\" % ( self.id, self.flickr_id, self.name, self.throttle_remaining )

class Photo(Entity): using_options(tablename = 'photo') flickr_id = Field(Text) title = Field(Text) url = Column(Text) pub_date = Column(DateTime) groups = ManyToMany('Group') user = ManyToOne('User') def __repr__(self): return \"<Photo('%s','%s','%s','%s')>\" % ( self.id, self.flickr_id, self.title, self.url )

engine = create_engine( 'mysql://twl:pass@localhost/sqlobject', echo=True) Session = sessionmaker(bind = engine) session = Session() metadata = MetaData() metadata.bind='mysql://twl:pass@localhost/sqlobject' metadata.bind.echo = True setup_all() for u in session.query(User): print u for g in session.query(Group): print g for p in session.query(Photo): print print

SqlSoup from sqlalchemy.ext.sqlsoup import SqlSoup soup = SqlSoup('mysql://twl:pass@localhost/ sqlobject') soup.photo.all()

soup.photo.order_by(soup.photo.title).all() [MappedPhoto(id=71L,flickr_id='3286255071',title='', user_id=1L,url='http://farm4.static.flickr.com/ 3249/3286255071_ff168f220b.jpg',pub_date=datetime.da tetime(2009, 2, 16, 20, 47, 56)), MappedPhoto(id=72L,flickr_id='3287070244',title='',u ser_id=1L,url='http://farm4.static.flickr.com/ 3298/3287070244_87a2a1b3ed.jpg',pub_date=datetime.da tetime(2009, 2, 16, 20, 47, 22)), MappedPhoto(id=46L,flickr_id='3395647634',title='And y Dustman',user_id=1L,url='http:// farm4.static.flickr.com/ 3554/3395647634_cc0c9f5a0a.jpg',pub_date=datetime.da tetime(2009, 3, 29, 9, 7, 34)), MappedPhoto(id=73L,flickr_id='3277628077',title='Bai nbridge Island Chinese New Year 2009',user_id=1L,url='http:// farm4.static.flickr.com/ 3334/3277628077_78025002f5.jpg',pub_date=datetime.da tetime(2009, 2, 13, 23, 31, 52)),

Migrations  sqlalchemy-migrate  Version control a database  A repository of upgrade scripts

from sqlalchemy import * from migrate import * metadata = MetaData(migrate_engine) photos_table = Table('photo', metadata, Column('id', Integer, primary_key=True), Column('flickr_id', Text), Column('title', Text), Column('user_id', Integer), Column('url', Text), Column('pub_date', DateTime), ) visible_col = Column('visible', Integer) def upgrade(): visible_col.create(photos_table) assert visible_col is photos_table.c.visible def downgrade(): visible_col.drop()

from sqlalchemy import * from migrate import * metadata = MetaData(migrate_engine) photos_table = Table('photo', metadata, Column('id', Integer, primary_key=True), Column('flickr_id', Text), Column('title', Text), Column('user_id', Integer), Column('url', Text), Column('pub_date', DateTime), Column('visible', Integer) ) col = photos_table.c.visible def upgrade(): col.alter(type=Boolean, nullable=False) def downgrade(): col.alter(type=Integer, nullable=True)

Migration operations  Create/Drop a table  Create/Drop/Alter a column  Create/Drop index  Create/Drop primary/foreign key constraints

Summary  Django for Django  SQLAlchemy for the rest

Thanks!  ted.leung@sun.com  twitter: twleung

MySQL User Conference 2009: Python and MySQL

0 comments

Notes on slide 1

1 Favorite