Successfully reported this slideshow.
We use your LinkedIn profile and activity data to personalize ads and to show you more relevant ads. You can change your ad preferences anytime.

Designing Opeation Oriented Web Applications / YAPC::Asia Tokyo 2011

6,579 views

Published on

Published in: Technology, Business
  • Be the first to comment

Designing Opeation Oriented Web Applications / YAPC::Asia Tokyo 2011

  1. 1. print STDERR “timeoutn”;timeoutwarn(“timeout”);timeout at test.pl line 8.
  2. 2. sub logf { my ($level,$message) = @_; my ($pkg,$file,$line) = caller; my @time = localtime; warn sprintf "%04d-%02d-%02dT%02d:%02d:%02d [%s] %s at %s line %d.n", $time[5]+1900, $time[4]+1, @time[3,2,1,0], $level, $message, $file, $line;}logf(“WARN”,”timeout”);logf(“INFO”,”foo bar”);2011-10-05T09:41:30 [WARN] timeout at test.pl line 10.2011-10-05T09:41:30 [INFO] foo bar at test.pl line 11.
  3. 3.
  4. 4. 2011-10-05T09:41:30[WARN][/entry/foobar]failed request: http://ad.net/ read timeoutat /path/to/Controler/Entry.pm line 10.
  5. 5. use Log::Minimal;# stack tracedebugf(‘blah blah blah’);infof(“server start $$”);critf("%s","foo");warnf("%d %s", 1, "foo");# stack tracedebugff(‘blah blah blah’);infoff(“server start $$”);critff("%s","foo");warnff("%d %s", 1, "foo");
  6. 6. use Log::Minimal;debugf(‘blah blah blah’); #local $ENV{LM_DEBUG} = 1;debugf(‘blah blah blah’); #2011-10-05T09:41:30 [DEBUG] blah.. at test.pl line 11.
  7. 7. #local $Log::Minimal::PRINT = sub { my ( $time, $type, $message, $trace) = @_; warn(“$time [$type] $message $tracen”);};#open( my $fh, ">>", "app.log");local $Log::Minimal::PRINT = sub { my ( $time, $type, $msg, $tr, $r_msg ) = @_; print $fh “$time [$type] $r_msg $trn”;};
  8. 8. #local $Log::Minimal::AUTODUMP = 1;warnf(“response => %s”,$res);2011-05-11T15:56:14 [WARN] response => [200,[Content-Type,text/plain],[OK]] at ..#local $Log::Minimal::COLOR = 1;warnf(“hirose31”);2011-05-11T15:56:14 [WARN] hirose31 at ..
  9. 9. use Log::Minimal;use Plack::Builder;builder {    enable "Log::Minimal", autodump => 1;    sub {        my $env = shift;        warnf("warn");        debugf("debug");        ...    }};$ plackup -a demo.psgi2011-05-11T16:32:24 [WARN] [/foo/bar/baz] warn at /tmp/demo.psgi line 82011-05-11T16:32:24 [DEBUG] [/foo/bar/baz] debug at /tmp/demo.psgi line 9
  10. 10. use Log::Minimal;use Parallel::Prefork;my $pm = Parallel::Prefork->new();while ( $pm->signal_received ne TERM ) { $pm->start(sub { local $Log::Minimal::AUTODUMP = 1; local $Log::Minimal::COLOR = 1; while (1) { my $job = $queue->dequeue; local $Log::Minimal::PRINT = sub { my ( $time, $type, $message, $trace) = @_; warn(“$$ [“,$job->name,”] [$type] $message $trace”); }; ... } });}
  11. 11. SELECT * FROM entries WHERE created_at >= 2011-10-05 00:00:00 AND created_at < 2011-10-06 00:00:00 AND status != 0;Data::Entry->search({ -and => [ created_at => {>=,2011-10-05 00:00:00 }, created_at => {<,2011-10-06 00:00:00 }, ], status => { !=, 0 }});
  12. 12. SELECT * FROM entries WHERE created_at >= 2011-10-05 00:00:00 AND created_at < 2011-10-06 00:00:00 AND status != 0;Data::Entry->search({ -and => [ created_at => {>=,2011-10-05 00:00:00 }, created_at => {<,2011-10-06 00:00:00 }, ], status => { !=, 0 }});
  13. 13. SELECT * FROM entries WHERE created_at >= 2011-10-05 00:00:00 AND created_at < 2011-10-06 00:00:00 AND status != 0;Data::Entry->search({ -and => [ created_at => {>=,2011-10-05 00:00:00 }, created_at => {<,2011-10-06 00:00:00 }, ], status => { !=, 0 }});
  14. 14. my $dbh = DBIx::Sunny->connect( ‘dsn’,‘user’,‘passwd’,);✓ RaiseError => 1✓ PrintError= > 0✓ ShowErrorStatement => 1✓ AutoInactiveDestroy => 1✓ sqlite_unicode/mysql_enable_utf8 => 1✓ mysql_auto_reconnect => 0
  15. 15. my $sth = $dbh->prepare(“SELECT * FROM entries WHERE ..”);SELECT /* test.pl line 941 */ * FROM entries WHERE ..
  16. 16. selectrow_arrayref($query, {}, @bind);selectrow_hashref($query, {}, @bind);selectall_arrayref($query, { Slice => {} }, @bind);prepare($query) && execute(@bind)
  17. 17. my $builder = SQL::Maker->new( driver => mysql,);my $dbh = DBIx::Sunny->connect(...);my ($sql, @binds) = $builder->select( user, [*], {name => john}, {order_by => user_id DESC});my $rows = $dbh->select_row($sql,@binds);
  18. 18. my $dbh = DBI->connect($dsn,$user,$pass,{...});while ( my $job = $queue->dequeue ) { ....}my $DBH;sub get_dbh { $DBH ||= DBI->connect($dsn,$user,$pass,{...}); $DBH;}while ( my $job = $queue->dequeue ) { my $dbh = get_dbh(); ....}sub get_dbh { DBI->connect_cached($dsn,$user,$pass,{...});}while ( my $job = $queue->dequeue ) { my $dbh = get_dbh(); ....}
  19. 19. my $dbh = DBI->connect($dsn,$user,$pass,{...});while ( my $job = $queue->dequeue ) { ....}my $DBH;sub get_dbh { $DBH ||= DBI->connect($dsn,$user,$pass,{...}); $DBH;}while ( my $job = $queue->dequeue ) { my $dbh = get_dbh(); ....}sub get_dbh { DBI->connect_cached($dsn,$user,$pass,{...});}while ( my $job = $queue->dequeue ) { my $dbh = get_dbh(); ....}
  20. 20. > SHOW PROCESSLIST;+--------+-------------+--------------------+---------+---------+-------+----------+------+| Id | User | Host | db | Command | Time | State | Info |+--------+-------------+--------------------+---------+---------+-------+----------+------+| 1 | system user | | NULL | Connect | 4851 | Wait... | NULL || 2 | system user | | NULL | Connect | 1 | Has r... | NULL || 235873 | www-slave | 10.0.xxx.39:33018 | cluster | Sleep | 8 | | NULL || 278334 | www-slave | 10.0.xxx.39:37627 | cluster | Sleep | 1206 | | NULL || 365403 | www-slave | 10.0.xxx.39:43957 | cluster | Sleep | 1259 | | NULL || 378019 | www-slave | 10.0.xxx.39:45098 | cluster | Sleep | 138 | | NULL || 397336 | www-slave | 10.0.xxx.39:48595 | cluster | Sleep | 377 | | NULL || 404611 | www-slave | 10.0.xxx.206:60525 | cluster | Sleep | 44 | | NULL || 405048 | www-slave | 10.0.xxx.206:60645 | cluster | Sleep | 119 | | NULL || 405091 | www-slave | 10.0.xxx.58:51982 | cluster | Sleep | 1104 | | NULL |.....811 rows in set (0.00 sec)
  21. 21. my $sc = start_scope_container(); #scope_container(‘key1’,‘value1’);$value = scope_container(‘key1’); #valuesub hoge { scope_container(‘key2’,‘foobar’);}{ my $sc2 = start_scope_container(); # $value = scope_container(‘key1’); #value scope_container(‘key1’,‘blahblah’); hoge(); $value = scope_container(‘key1’); #blahblah $value = scope_container(‘key2’); #foobar undef $sc2;}$value = scope_container(‘key1’); #value$value = scope_container(‘key2’); #undefined
  22. 22. my %DATA; #$DATA{key1} = value1;$value = $DATA{key1} #valuesub hoge { $DATA{key2} = foobar;}{ local %DATA; # $value = $DATA{key1} #value $DATA{key1} = ‘blahblah’ hoge(); $value = $DATA{key1}; #blahblah $value = $DATA{key2} #foobar}$value = $DATA{key1}; #value$value = $DATA{key2}; #undefined
  23. 23. sub get_dbh { my $dsn = shift my $cache = scope_container($dsn); reutrn $cache if $cache; my $dbh = DBIx:Sunny->connect($dsn,$user,$pass); scope_container($dsn, $dbh); return $dbh;}while ( my $job = $queue->dequeue ) { my $sc = start_scope_container(); my $dbh = get_dbh(); #new connection my $dbh2 = get_dbh(); #cached .. # disconnect}
  24. 24. use Scope::Container;use Scope::Container::DBI;sub get_dbh { Scope::Container::DBI->connect($dsn,$user,$pass,{}):}for my $id (1..$max) { my $sc = start_scope_container(); get_dbh()->do(...); #new connection get_dbh()->select_row(...); #cache undef $sc; #}
  25. 25. # DBIlocal $Scope::Container::DBI::DBI_CLASS = “Amon2::DBI”;my $dbh = Scope::Container::DBI->connect( $dsn,$user,$pass,{});# slave 1my $dbh = Scope::Container::DBI->connect( [$dsn1,$user,$pass,{}], [$dsn2,$user,$pass,{}], [$dsn3,$user,$pass,{}],);
  26. 26. use Plack::Builder;builder { enable “Scope::Container”; $app;};# Plack::Middleware::Scope::Containersub call { my ( $self, $env) = @_; my $container = start_scope_container(); $self->app->($env);}
  27. 27. key1 key2 key3 key4 key5 key6 key7 key8 key9 memcached1 memcached2 memcached3
  28. 28. key1 key2 key3 key4 key5 key6 key7 key8 key9 memcached1 memcached2 memcached3
  29. 29. key1 key2 key3 key4 key5 key6 key7 key8 key9 memcached1 memcached2 memcached3
  30. 30. key1 key2 key3 key4 key5 key6 key7 key8 key9 memcached1 memcached2 memcached3
  31. 31. App Appmem DB
  32. 32. App App App Appmem DB mem
  33. 33. App App App Appmem DB mem DB
  34. 34. use Cache::Memcached::IronPlate;use Cache::Memcached::Fast;my $memd = Cache::Memcached::IronPlate->new( cache => Cache::Memcached::Fast->new(...),);$memd->get$memd->get_multi$memd->set$memd->add$memd->replace$memd->append$memd->prepend$memd->incr$memd->counter$memd->decr$memd->delete
  35. 35. set(‘key:dup’,val,exp) get(‘key:dup’) IronPlate IronPlate set key:dist:{1..3} get key:dist:{1..3}mem1 mem2 mem3 mem1 mem2 mem3
  36. 36. set(‘key:dist’,val,exp) get(‘key:dist’) IronPlate IronPlate set key:dist:{1..30} get key:dist:rand(30) mem1 mem2 mem3 mem1 mem2 mem3
  37. 37. use Cache::Isolator;use Cache::Memcached::Fast; my $isolator = Cache::Isolator->new(    cache => Cache::Memcached::Fast->new(...),);$isolator->get(key);$isolator->set(key,val,expires);
  38. 38. set(‘key’,val,30) set(‘key’,val,30) Isolator Isolatorset key 30 set key:earlyexp 20 mem1 mem2 mem1 mem2
  39. 39. use Cache::Isolator;use Cache::Memcached::Fast; my $isolator = Cache::Isolator->new(    cache => Cache::Memcached::Fast->new(...), early_expires_ratio => 10, expires_before => 10);$isolator->get(key);$isolator->set(key,val,expires);
  40. 40. use Cache::Isolator;use Cache::Memcached::Fast; my $isolator = Cache::Isolator->new(    cache => Cache::Memcached::Fast->new(...),    concurrency => 3,);$isolator->get_or_set(key, sub { my $dbh = get_dbh(); $dbh->select_all(heavy_sql);}, expires);
  41. 41. DBresponse memcached
  42. 42. DBresponse memcached
  43. 43. DBresponse memcached
  44. 44. DBresponse memcached
  45. 45. DBresponse memcached
  46. 46. DBresponse memcached
  47. 47. DBresponse memcached
  48. 48. DBresponse memcached
  49. 49. DBresponse memcached
  50. 50. DBresponse memcached
  51. 51. DBresponse memcached
  52. 52. DBresponse memcached
  53. 53. use Plack::Builder;use File::Temp;builder { enable "ServerStatus::Lite", path => /___server-status, allow => [‘127.0.0.1’,‘192.168.0.0/16’], scoreboard => File::Temp::tempdir(CLEANUP => 1); $app};
  54. 54. $ curl http://localhost:5000/___server-statusUptime: 1318372692BusyWorkers: 3IdleWorkers: 27--pid status remote_addr host method uri protocol1522 A 127.0.0.1 localhost:5000 GET /___server-status HTTP/1.11523 _ 192.168.67.3 192.168.67.4:5000 GET /foo/bar HTTP/1.01526 _ 192.168.67.5 192.168.67.4:5000 GET /foo/bar HTTP/1.0...
  55. 55. use Parallel::Prefork;use Parallel::Scoreboard;my $sb = Parallel::Scoreboard->new( base_dir => ‘/var/run/example_worker’ );my $pm = Parallel::Prefork->new({...});while ( $pm->signal_received !~ m!^(?:TERM|INT)$! ) { $pm->start(sub{ $sb->update(.); while(1) { my $job = $queue->dequeue; $sb->update(A); #work work work $sb->update(.); } });}
  56. 56. use Parallel::Prefork;use Parallel::Scoreboard;my $sb = Parallel::Scoreboard->new( base_dir => ‘/var/run/example_worker’ );my $pm = Parallel::Prefork->new({...});while ( $pm->signal_received !~ m!^(?:TERM|INT)$! ) { $pm->start(sub{ idle $sb->update(.); while(1) { my $job = $queue->dequeue; $sb->update(A); #work work work busy $sb->update(.); } }); idle}
  57. 57. # scoreboard.psgimy $sb = Parallel::Scoreboard->new( base_dir => ‘/var/run/example_worker’ );sub { my $env = shift; my $stats = $scoreboard->read_all(); my $busy = 0; my $idle = 0; for my $pid ( sort { $a <=> $b } keys %$stats) { if ( $stats->{$pid} =~ m!^A! ) { $busy++; } else { busy $idle++; } } return [200,[Content-Type=>plain/text], ["BusyWorkers: $busynIdleWorkers: $idle"]];};
  58. 58. #$ plackup -p 5001 -a scoreboard.psgi#$ curl http://localhost:5001/___server-statusBusyWorkers: 6IdleWorkers: 4
  59. 59. 192.188.67.2 idle busy

×