Successfully reported this slideshow.
We use your LinkedIn profile and activity data to personalize ads and to show you more relevant ads. You can change your ad preferences anytime.

MySQL User Conference 2009: Python and MySQL

2,118 views

Published on

Published in: Technology, Sports
  • Be the first to comment

MySQL User Conference 2009: Python and MySQL

  1. 1. Python and MySQL Ted Leung Sun Microsystems
  2. 2. Python-DB  In the beginning there was the DB-API  PEP-249
  3. 3. MySQLdb  MySQLdb written in Python  _mysql written in C and is MySQL specific
  4. 4. import MySQLdb conn = MySQLdb.connect( host = 'localhost', user = 'twl', Text Text Text passwd='pass', Text db='flickrmgr' )
  5. 5. c = conn.cursor() c.execute(quot;quot;quot;SELECT * from curator_photoquot;quot;quot;) # read one row pprint(c.fetchone()) # read all the rows pprint(c.fetchall()) ((3L, 3422686825L, quot;OPG Toymaker's Doll 2009quot;, 2L, 'http://farm4.static.flickr.com/ 3377/3422686825_fd57ea30e7.jpg', datetime.datetime(2009, 4, 8, 1, 21, 44)), (4L, 3422685683L, quot;OPG Toymaker's Doll 2009quot;, 2L, 'http://farm4.static.flickr.com/ 3608/3422685683_4d16829c19.jpg', datetime.datetime(2009, 4, 8, 1, 20, 54)),
  6. 6. dc = conn.cursor(MySQLdb.cursors.DictCursor) dc.execute(quot;quot;quot;SELECT * from curator_photoquot;quot;quot;) pprint(dc.fetchmany(5)) ({'flickr_id': 2147483647L, 'id': 2L, 'pub_date': datetime.datetime(2009, 4, 8, 1, 21, 44), 'title': quot;OPG Toymaker's Doll 2009quot;, 'url': 'http://farm4.static.flickr.com/ 3377/3422686825_fd57ea30e7.jpg', 'user_id': 2L}, {'flickr_id': 3422686825L, 'id': 3L, 'pub_date': datetime.datetime(2009, 4, 8, 1, 21, 44), 'title': quot;OPG Toymaker's Doll 2009quot;, 'url': 'http://farm4.static.flickr.com/ 3377/3422686825_fd57ea30e7.jpg', 'user_id': 2L})
  7. 7. c.execute( quot;quot;quot;SELECT * from curator_photo WHERE id < %squot;quot;quot;, (50,)) pprint(c.fetchall())
  8. 8. c.execute( quot;quot;quot;INSERT INTO curator_user (flickr_id, name) VALUES (%s, %s)quot;quot;quot;, (quot;000quot;, quot;No Userquot;) ) conn.commit()
  9. 9. Django  Leading web application framework  Lots of people coming to Python via Django today  Model-View-Controller  Command line management scripts  It’s own Rails-like ORM  Based on the Active Record pattern
  10. 10. Active Record Pattern  Database table is wrapped in a class  Each class instance is a row in the table  Relationships expressd as foreign key constraints
  11. 11. DATABASE_ENGINE = 'mysql' DATABASE_NAME = 'flickrmgr' DATABASE_USER = 'twl' DATABASE_PASSWORD = 'pass'
  12. 12. from django.db import models class User(models.Model): flickr_id = models.TextField() name = models.TextField() def __unicode__(self): return self.name class Group(models.Model): flickr_id = models.TextField() name = models.TextField() throttle_count = models.IntegerField() throttle_mode = models.TextField() throttle_remaining = models.IntegerField() def __unicode__(self): return self.name
  13. 13. class Photo(models.Model): flickr_id = models.IntegerField() title = models.TextField() user = models.ForeignKey(User) groups = models.ManyToManyField(Group) url = models.URLField() pub_date = models.DateTimeField() def __unicode__(self): return self.title
  14. 14. from curator.models import Photo from datetime import datetime Photo.objects.all() Photo.objects.filter(title__contains='PyCon') Photo.objects.filter(title__contains='PyCon').exclude( pub_date__lte=datetime(2009,4,1)) Photo.objects.filter(title__contains='PyCon').exclude( pub_date__lte=datetime(2009,4,1))[2:4]
  15. 15. from django.contrib import admin class PhotoAdmin(admin.ModelAdmin): pass class GroupAdmin(admin.ModelAdmin): pass class UserAdmin(admin.ModelAdmin): pass admin.site.register(Group, GroupAdmin) admin.site.register(Photo, PhotoAdmin) admin.site.register(User, UserAdmin)
  16. 16. def load_from_flickr(request): api = API() # 51035696189@N01 twl, created = User.objects.get_or_create( flickr_id = '51035696189@N01', name = 'Ted Leung' ) if created: twl.save() photos = api.list_user_info( 'http://www.flickr.com/photos/twleung')
  17. 17. for photo in photos: flickr_id, title, pub_date, url, pools = photo new_photo = Photo() new_photo.flickr_id = flickr_id new_photo.title = title new_photo.user = twl new_photo.pub_date = datetime.fromtimestamp(int(pub_date)) new_photo.url = url new_photo.save()
  18. 18. # do pools for pool in pools: new_pool, created=Group.objects.get_or_create( flickr_id = pool[0], name = pool[1], throttle_count = pool[2], throttle_mode = pool[3], throttle_remaining = pool[4] ) new_photo.groups.add(new_pool) if created: new_pool.save()
  19. 19. output = ''' <html> <head> <title>Bulk loading from Flickr</title> </head> <body> </body> </html> ''' return HttpResponse(output)
  20. 20. from django.conf.urls.defaults import * from curator.models import User, Group, Photo from curator.views import load_from_flickr photo_info_dict = { 'queryset': Photo.objects.all(), 'date_field': 'pub_date', } urlpatterns = patterns( '', (r'^$', 'django.views.generic.date_based.archive_index', photo_info_dict), (r'^load/$', load_from_flickr), )
  21. 21. Transactions  Commit on save or delete  Commit at HTTP request/response boundaries  commit_manually decorator
  22. 22. Connection Pooling  Important as systems scale up  No framework wide solution yet
  23. 23. Migration  South  Integrates with manage.py  Can automatically migrate models
  24. 24. ../bin/django startmigration curator --initial Creating migrations directory at '/Users/twl/work/ mysql-2009/django/flickrmgr/curator/migrations'... Creating __init__.py in '/Users/twl/work/mysql-2009/ django/flickrmgr/curator/migrations'... + Added model 'curator.Photo' + Added model 'curator.Group' + Added model 'curator.User' + Added field 'curator.Photo.groups' Created 0001_initial.py.
  25. 25. class Photo(models.Model): flickr_id = models.IntegerField() title = models.TextField() user = models.ForeignKey(User) groups = models.ManyToManyField(Group) url = models.URLField() pub_date = models.DateTimeField() visible = models.BooleanField()
  26. 26. ../bin/django startmigration curator add_visible --auto + Added field 'curator.photo.visible' Created 0002_add_visible.py.
  27. 27. from south.db import db from django.db import models from curator.models import * class Migration: def forwards(self, orm): # Adding field 'Photo.visible' db.add_column('curator_photo', 'visible', models.BooleanField()) def backwards(self, orm): # Deleting field 'Photo.visible' db.delete_column('curator_photo', 'visible')
  28. 28. SQLObject  ORM using Active Record pattern  Tight coupling to Python classes  Used in TurboGears 1
  29. 29. connection_string='mysql://twl:pass@localhost/sqlobject' connection=connectionForURI(connection_string) sqlhub.processConnection = connection try: User.createTable(ifNotExists=True) Photo.createTable(ifNotExists=True) FlickrGroup.createTable(ifNotExists=True) except dberrors.OperationalError, oe: print oe
  30. 30. class User(SQLObject): flickr_id = StringCol() name = StringCol() class Photo(SQLObject): flickr_id = StringCol() title = StringCol() user = ForeignKey('User') groups = RelatedJoin('FlickrGroup') url = StringCol() pub_date = DateTimeCol() class FlickrGroup(SQLObject): flickr_id = StringCol() name = StringCol() throttle_count = IntCol() throttle_mode = StringCol() throttle_remaining = IntCol() photos = RelatedJoin('Photo')
  31. 31. pycon_photos = list(Photo.select(Photo.q.title.contains('PyCon'))) pprint(pycon_photos) pycon_apr_photos = list(Photo.select(AND(Photo.q.title.contains('PyCon'), Photo.q.pub_date > datetime(2009,4,1) ))) pprint(pycon_apr_photos)
  32. 32. api = API() twl = User( flickr_id = '51035696189@N01', name = 'Ted Leung' ) photos = api.list_user_info( 'http://www.flickr.com/photos/twleung')
  33. 33. for photo in photos: print photo flickr_id, title, pub_date, url, pools = photo new_photo = Photo( flickr_id = flickr_id, title = title, user = twl, pub_date = datetime.fromtimestamp(int(pub_date)), url = url )
  34. 34. # do pools for pool in pools: new_pool = list(FlickrGroup.select( FlickrGroup.q.flickr_id == pool[0])) if len(new_pool) > 0: new_pool = new_pool[0] else: new_pool = None if not new_pool: new_pool = FlickrGroup( flickr_id = pool[0], name = pool[1], throttle_count = pool[2], throttle_mode = pool[3], throttle_remaining = pool[4] ) new_photo.addFlickrGroup(new_pool)
  35. 35. Transactions txn = connection.transaction() p = Photo.get(1, txn) p.title = ‘updated photo’ txn.commit()
  36. 36. Connection Pooling  Connection pooling is built in and on by default
  37. 37. sqlmeta class Photo(SQLObject): class sqlmeta: lazyUpdate = True cacheValues = False flickr_id = StringCol() title = StringCol() user = ForeignKey('User') groups = RelatedJoin('FlickrGroup') url = StringCol() pub_date = DateTimeCol()
  38. 38. Events from sqlobject.events import listen from sqlobject.events RowUpdateSignal, RowCreatedSignal def update_listener(instance, kwargs): kwargs['pub_date'] = datetime.datetime.now() def created_listener(kwargs, post_funcs): print “created photo %s” % (kwargs[‘title’]) listen(update_listener, Photo, RowUpdateSignal) listen(created_listener, Photo, RowCreatedSignal)
  39. 39. SQLAlchemy  Python’s answer to Hibernate  Low level SQL manipulations  High Level ORM  Used in TurboGears 2
  40. 40. Low Level SQL
  41. 41. from sqlalchemy import create_engine from sqlalchemy import Table, Column, Integer, String, Text, DateTime, MetaData, ForeignKey from sqlalchemy import select engine = create_engine( 'mysql://twl:pass@localhost/sqlobject',echo=True) metadata = MetaData()
  42. 42. users_table = Table('user', metadata, Column('id', Integer, primary_key=True), Column('flickr_id', Text), Column('name', Text) ) groups_table = Table('group', metadata, Column('id', Integer, primary_key=True), Column('flickr_id', Text), Column('name', Text), Column('throttle_count', Integer), Column('throttle_mode', Text), Column('throttle_remaining', Integer) ) photos_table = Table('photo', metadata, Column('id', Integer, primary_key=True), Column('flickr_id', Text), Column('title', Text), Column('user_id', Integer, ForeignKey(‘user.id’)), Column('url', Text), Column('pub_date', DateTime) )
  43. 43. metadata.create_all(engine) conn = engine.connect() s = select([users_table]) result = conn.execute(s) for row in result: print row s = select([photos_table], photos_table.c.title.contains('PyCon')) for row in conn.execute(s): print row
  44. 44. ins = users.insert().values(name='Thomas Hawk', flickr_id='000') result = conn.execute(ins) update = users.update().where( users.c.name=='Thomas Hawk' ).values( flickr_id='51035555243@N01')) result = conn.execute(update)
  45. 45. Transactions  Regular manual control  Autocommit - per session  2PC
  46. 46. Connection Pooling  Built-in framework  Done at engine configuration time
  47. 47. Other Low Level SQL features  Unions and other Set operations  Scalar Selects  Correlated Subqueries  Ordering, Grouping, Offsetting  Correlated Updates
  48. 48. ORM  Mapping  Declarative
  49. 49. class User(object): def __repr__(self): return quot;<User('%s','%s','%s')>quot; % ( self.id, self.flickr_id, self.name) class Group(object): def __repr__(self): return quot;<Group('%s','%s','%s','%s')>quot; % ( self.id, self.flickr_id, self.name, self.throttle_remaining ) class Photo(object): def __repr__(self): return quot;<Photo('%s','%s','%s','%s')>quot; % ( self.id, self.flickr_id, self.title, self.url )
  50. 50. metadata = MetaData() flickr_group_photo = Table( 'flickr_group_photo', metadata, Column('flickr_group_id', Integer, ForeignKey('flickr_group.id')), Column('photo_id', Integer, ForeignKey('photo.id')))
  51. 51. from sqlalchemy.orm import mapper, sessionmaker engine = create_engine( 'mysql://twl:pass@localhost/sqlobject')
  52. 52. Session = sessionmaker(bind = engine) session = Session() mapper(User, users_table, properties = { 'photos' : relation(Photo, backref='user') }) mapper(Group, groups_table, properties = { 'photos' : relation(Photo, secondary=flickr_group_photo, backref='groups') }) mapper(Photo, photos_table) for u in session.query(User): print u for g in session.query(Group): print g for p in session.query(Photo): print p
  53. 53. Declarative ORM
  54. 54. from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import sessionmaker from sqlalchemy import Column, ForeignKey from sqlalchemy import Integer, String, Text, DateTime Base = declarative_base() class User(Base): __tablename__ = 'user' id = Column(Integer, primary_key=True) flickr_id = Column(Text) name = Column(Text) def __repr__(self): return quot;<User('%s','%s','%s')>quot; % ( self.id, self.flickr_id, self.name)
  55. 55. class Group(Base): __tablename__ = 'flickr_group' id = Column(Integer, primary_key=True) flickr_id = Column(Text) name = Column(Text) throttle_count = Column(Integer) throttle_mode = Column(Text) throttle_remaining = Column(Integer) def __repr__(self): return quot;<Group('%s','%s','%s','%s')>quot; % ( self.id, self.flickr_id, self.name, self.throttle_remaining )
  56. 56. class Photo(Base): __tablename__ = 'photo' id = Column(Integer, primary_key=True) flickr_id = Column(Text) title = Column(Text) user_id = Column(Integer,ForeignKey('user.id')) user = relation(User, backref=backref('photos', order_by=id)) groups = relation('Group', secondary=flickr_group_photo, backref=backref('photos')) url = Column(Text) pub_date = Column(DateTime) def __repr__(self): return quot;<Photo('%s','%s','%s','%s')>quot; % ( self.id, self.flickr_id, self.title, self.url )
  57. 57. Session = sessionmaker(bind = engine) session = Session() for u in session.query(User): print u for g in session.query(Group): print g for p in session.query(Photo): print p
  58. 58. And more  Advanced mapping  Multiple table mapping
  59. 59. Elixir  a simpler way of doing declarative ORM for SQLAlchemy
  60. 60. from elixir import * class User(Entity): using_options(tablename = 'user') flickr_id = Field(Text) name = Field(Text) photos = OneToMany('Photo') def __repr__(self): return quot;<User('%s','%s','%s')>quot; % ( self.id, self.flickr_id, self.name)
  61. 61. class Group(Entity): using_options(tablename = 'flickr_group') flickr_id = Field(Text) name = Field(Text) throttle_count = Field(Integer) throttle_mode = Field(Text) throttle_remaining = Field(Integer) def __repr__(self): return quot;<Group('%s','%s','%s','%s')>quot; % ( self.id, self.flickr_id, self.name, self.throttle_remaining )
  62. 62. class Photo(Entity): using_options(tablename = 'photo') flickr_id = Field(Text) title = Field(Text) url = Column(Text) pub_date = Column(DateTime) groups = ManyToMany('Group') user = ManyToOne('User') def __repr__(self): return quot;<Photo('%s','%s','%s','%s')>quot; % ( self.id, self.flickr_id, self.title, self.url )
  63. 63. engine = create_engine( 'mysql://twl:pass@localhost/sqlobject', echo=True) Session = sessionmaker(bind = engine) session = Session() metadata = MetaData() metadata.bind='mysql://twl:pass@localhost/sqlobject' metadata.bind.echo = True setup_all() for u in session.query(User): print u for g in session.query(Group): print g for p in session.query(Photo): print print
  64. 64. SqlSoup from sqlalchemy.ext.sqlsoup import SqlSoup soup = SqlSoup('mysql://twl:pass@localhost/ sqlobject') soup.photo.all()
  65. 65. soup.photo.order_by(soup.photo.title).all() [MappedPhoto(id=71L,flickr_id='3286255071',title='', user_id=1L,url='http://farm4.static.flickr.com/ 3249/3286255071_ff168f220b.jpg',pub_date=datetime.da tetime(2009, 2, 16, 20, 47, 56)), MappedPhoto(id=72L,flickr_id='3287070244',title='',u ser_id=1L,url='http://farm4.static.flickr.com/ 3298/3287070244_87a2a1b3ed.jpg',pub_date=datetime.da tetime(2009, 2, 16, 20, 47, 22)), MappedPhoto(id=46L,flickr_id='3395647634',title='And y Dustman',user_id=1L,url='http:// farm4.static.flickr.com/ 3554/3395647634_cc0c9f5a0a.jpg',pub_date=datetime.da tetime(2009, 3, 29, 9, 7, 34)), MappedPhoto(id=73L,flickr_id='3277628077',title='Bai nbridge Island Chinese New Year 2009',user_id=1L,url='http:// farm4.static.flickr.com/ 3334/3277628077_78025002f5.jpg',pub_date=datetime.da tetime(2009, 2, 13, 23, 31, 52)),
  66. 66. Migrations  sqlalchemy-migrate  Version control a database  A repository of upgrade scripts
  67. 67. from sqlalchemy import * from migrate import * metadata = MetaData(migrate_engine) photos_table = Table('photo', metadata, Column('id', Integer, primary_key=True), Column('flickr_id', Text), Column('title', Text), Column('user_id', Integer), Column('url', Text), Column('pub_date', DateTime), ) visible_col = Column('visible', Integer) def upgrade(): visible_col.create(photos_table) assert visible_col is photos_table.c.visible def downgrade(): visible_col.drop()
  68. 68. from sqlalchemy import * from migrate import * metadata = MetaData(migrate_engine) photos_table = Table('photo', metadata, Column('id', Integer, primary_key=True), Column('flickr_id', Text), Column('title', Text), Column('user_id', Integer), Column('url', Text), Column('pub_date', DateTime), Column('visible', Integer) ) col = photos_table.c.visible def upgrade(): col.alter(type=Boolean, nullable=False) def downgrade(): col.alter(type=Integer, nullable=True)
  69. 69. Migration operations  Create/Drop a table  Create/Drop/Alter a column  Create/Drop index  Create/Drop primary/foreign key constraints
  70. 70. Summary  Django for Django  SQLAlchemy for the rest
  71. 71. Thanks!  ted.leung@sun.com  twitter: twleung

×