Designing Opeation Oriented Web Applications / YAPC::Asia Tokyo 2011

print STDERR “timeoutn”;timeoutwarn(“timeout”);timeout at test.pl line 8.

sub logf { my ($level,$message) = @_; my ($pkg,$file,$line) = caller; my @time = localtime; warn sprintf "%04d-%02d-%02dT%02d:%02d:%02d [%s] %s at %s line %d.n", $time[5]+1900, $time[4]+1, @time[3,2,1,0], $level, $message, $file, $line;}logf(“WARN”,”timeout”);logf(“INFO”,”foo bar”);2011-10-05T09:41:30 [WARN] timeout at test.pl line 10.2011-10-05T09:41:30 [INFO] foo bar at test.pl line 11.

2011-10-05T09:41:30[WARN][/entry/foobar]failed request: http://ad.net/ read timeoutat /path/to/Controler/Entry.pm line 10.

use Log::Minimal;# stack tracedebugf(‘blah blah blah’);infof(“server start $$”);critf("%s","foo");warnf("%d %s", 1, "foo");# stack tracedebugff(‘blah blah blah’);infoff(“server start $$”);critff("%s","foo");warnff("%d %s", 1, "foo");

use Log::Minimal;debugf(‘blah blah blah’); #local $ENV{LM_DEBUG} = 1;debugf(‘blah blah blah’); #2011-10-05T09:41:30 [DEBUG] blah.. at test.pl line 11.

#local $Log::Minimal::PRINT = sub { my ( $time, $type, $message, $trace) = @_; warn(“$time [$type] $message $tracen”);};#open( my $fh, ">>", "app.log");local $Log::Minimal::PRINT = sub { my ( $time, $type, $msg, $tr, $r_msg ) = @_; print $fh “$time [$type] $r_msg $trn”;};

#local $Log::Minimal::AUTODUMP = 1;warnf(“response => %s”,$res);2011-05-11T15:56:14 [WARN] response => [200,[Content-Type,text/plain],[OK]] at ..#local $Log::Minimal::COLOR = 1;warnf(“hirose31”);2011-05-11T15:56:14 [WARN] hirose31 at ..

use Log::Minimal;use Plack::Builder;builder { enable "Log::Minimal", autodump => 1; sub { my $env = shift; warnf("warn"); debugf("debug"); ... }};$ plackup -a demo.psgi2011-05-11T16:32:24 [WARN] [/foo/bar/baz] warn at /tmp/demo.psgi line 82011-05-11T16:32:24 [DEBUG] [/foo/bar/baz] debug at /tmp/demo.psgi line 9

use Log::Minimal;use Parallel::Prefork;my $pm = Parallel::Prefork->new();while ( $pm->signal_received ne TERM ) { $pm->start(sub { local $Log::Minimal::AUTODUMP = 1; local $Log::Minimal::COLOR = 1; while (1) { my $job = $queue->dequeue; local $Log::Minimal::PRINT = sub { my ( $time, $type, $message, $trace) = @_; warn(“$$ [“,$job->name,”] [$type] $message $trace”); }; ... } });}

SELECT * FROM entries WHERE created_at >= 2011-10-05 00:00:00 AND created_at < 2011-10-06 00:00:00 AND status != 0;Data::Entry->search({ -and => [ created_at => {>=,2011-10-05 00:00:00 }, created_at => {<,2011-10-06 00:00:00 }, ], status => { !=, 0 }});

my $dbh = DBIx::Sunny->connect( ‘dsn’,‘user’,‘passwd’,);✓ RaiseError => 1✓ PrintError= > 0✓ ShowErrorStatement => 1✓ AutoInactiveDestroy => 1✓ sqlite_unicode/mysql_enable_utf8 => 1✓ mysql_auto_reconnect => 0

my $sth = $dbh->prepare(“SELECT * FROM entries WHERE ..”);SELECT /* test.pl line 941 */ * FROM entries WHERE ..

selectrow_arrayref($query, {}, @bind);selectrow_hashref($query, {}, @bind);selectall_arrayref($query, { Slice => {} }, @bind);prepare($query) && execute(@bind)

my $builder = SQL::Maker->new( driver => mysql,);my $dbh = DBIx::Sunny->connect(...);my ($sql, @binds) = $builder->select( user, [*], {name => john}, {order_by => user_id DESC});my $rows = $dbh->select_row($sql,@binds);

my $dbh = DBI->connect($dsn,$user,$pass,{...});while ( my $job = $queue->dequeue ) { ....}my $DBH;sub get_dbh { $DBH ||= DBI->connect($dsn,$user,$pass,{...}); $DBH;}while ( my $job = $queue->dequeue ) { my $dbh = get_dbh(); ....}sub get_dbh { DBI->connect_cached($dsn,$user,$pass,{...});}while ( my $job = $queue->dequeue ) { my $dbh = get_dbh(); ....}

> SHOW PROCESSLIST;+--------+-------------+--------------------+---------+---------+-------+----------+------+| Id | User | Host | db | Command | Time | State | Info |+--------+-------------+--------------------+---------+---------+-------+----------+------+| 1 | system user | | NULL | Connect | 4851 | Wait... | NULL || 2 | system user | | NULL | Connect | 1 | Has r... | NULL || 235873 | www-slave | 10.0.xxx.39:33018 | cluster | Sleep | 8 | | NULL || 278334 | www-slave | 10.0.xxx.39:37627 | cluster | Sleep | 1206 | | NULL || 365403 | www-slave | 10.0.xxx.39:43957 | cluster | Sleep | 1259 | | NULL || 378019 | www-slave | 10.0.xxx.39:45098 | cluster | Sleep | 138 | | NULL || 397336 | www-slave | 10.0.xxx.39:48595 | cluster | Sleep | 377 | | NULL || 404611 | www-slave | 10.0.xxx.206:60525 | cluster | Sleep | 44 | | NULL || 405048 | www-slave | 10.0.xxx.206:60645 | cluster | Sleep | 119 | | NULL || 405091 | www-slave | 10.0.xxx.58:51982 | cluster | Sleep | 1104 | | NULL |.....811 rows in set (0.00 sec)

my $sc = start_scope_container(); #scope_container(‘key1’,‘value1’);$value = scope_container(‘key1’); #valuesub hoge { scope_container(‘key2’,‘foobar’);}{ my $sc2 = start_scope_container(); # $value = scope_container(‘key1’); #value scope_container(‘key1’,‘blahblah’); hoge(); $value = scope_container(‘key1’); #blahblah $value = scope_container(‘key2’); #foobar undef $sc2;}$value = scope_container(‘key1’); #value$value = scope_container(‘key2’); #undefined

my %DATA; #$DATA{key1} = value1;$value = $DATA{key1} #valuesub hoge { $DATA{key2} = foobar;}{ local %DATA; # $value = $DATA{key1} #value $DATA{key1} = ‘blahblah’ hoge(); $value = $DATA{key1}; #blahblah $value = $DATA{key2} #foobar}$value = $DATA{key1}; #value$value = $DATA{key2}; #undefined

sub get_dbh { my $dsn = shift my $cache = scope_container($dsn); reutrn $cache if $cache; my $dbh = DBIx:Sunny->connect($dsn,$user,$pass); scope_container($dsn, $dbh); return $dbh;}while ( my $job = $queue->dequeue ) { my $sc = start_scope_container(); my $dbh = get_dbh(); #new connection my $dbh2 = get_dbh(); #cached .. # disconnect}

use Scope::Container;use Scope::Container::DBI;sub get_dbh { Scope::Container::DBI->connect($dsn,$user,$pass,{}):}for my $id (1..$max) { my $sc = start_scope_container(); get_dbh()->do(...); #new connection get_dbh()->select_row(...); #cache undef $sc; #}

# DBIlocal $Scope::Container::DBI::DBI_CLASS = “Amon2::DBI”;my $dbh = Scope::Container::DBI->connect( $dsn,$user,$pass,{});# slave 1my $dbh = Scope::Container::DBI->connect( [$dsn1,$user,$pass,{}], [$dsn2,$user,$pass,{}], [$dsn3,$user,$pass,{}],);

use Plack::Builder;builder { enable “Scope::Container”; $app;};# Plack::Middleware::Scope::Containersub call { my ( $self, $env) = @_; my $container = start_scope_container(); $self->app->($env);}

key1 key2 key3 key4 key5 key6 key7 key8 key9 memcached1 memcached2 memcached3

App Appmem DB

App App App Appmem DB mem

App App App Appmem DB mem DB

use Cache::Memcached::IronPlate;use Cache::Memcached::Fast;my $memd = Cache::Memcached::IronPlate->new( cache => Cache::Memcached::Fast->new(...),);$memd->get$memd->get_multi$memd->set$memd->add$memd->replace$memd->append$memd->prepend$memd->incr$memd->counter$memd->decr$memd->delete

set(‘key:dup’,val,exp) get(‘key:dup’) IronPlate IronPlate set key:dist:{1..3} get key:dist:{1..3}mem1 mem2 mem3 mem1 mem2 mem3

set(‘key:dist’,val,exp) get(‘key:dist’) IronPlate IronPlate set key:dist:{1..30} get key:dist:rand(30) mem1 mem2 mem3 mem1 mem2 mem3

use Cache::Isolator;use Cache::Memcached::Fast; my $isolator = Cache::Isolator->new( cache => Cache::Memcached::Fast->new(...),);$isolator->get(key);$isolator->set(key,val,expires);

set(‘key’,val,30) set(‘key’,val,30) Isolator Isolatorset key 30 set key:earlyexp 20 mem1 mem2 mem1 mem2

use Cache::Isolator;use Cache::Memcached::Fast; my $isolator = Cache::Isolator->new( cache => Cache::Memcached::Fast->new(...), early_expires_ratio => 10, expires_before => 10);$isolator->get(key);$isolator->set(key,val,expires);

use Cache::Isolator;use Cache::Memcached::Fast; my $isolator = Cache::Isolator->new( cache => Cache::Memcached::Fast->new(...), concurrency => 3,);$isolator->get_or_set(key, sub { my $dbh = get_dbh(); $dbh->select_all(heavy_sql);}, expires);

DBresponse memcached

use Plack::Builder;use File::Temp;builder { enable "ServerStatus::Lite", path => /___server-status, allow => [‘127.0.0.1’,‘192.168.0.0/16’], scoreboard => File::Temp::tempdir(CLEANUP => 1); $app};

$ curl http://localhost:5000/___server-statusUptime: 1318372692BusyWorkers: 3IdleWorkers: 27--pid status remote_addr host method uri protocol1522 A 127.0.0.1 localhost:5000 GET /___server-status HTTP/1.11523 _ 192.168.67.3 192.168.67.4:5000 GET /foo/bar HTTP/1.01526 _ 192.168.67.5 192.168.67.4:5000 GET /foo/bar HTTP/1.0...

use Parallel::Prefork;use Parallel::Scoreboard;my $sb = Parallel::Scoreboard->new( base_dir => ‘/var/run/example_worker’ );my $pm = Parallel::Prefork->new({...});while ( $pm->signal_received !~ m!^(?:TERM|INT)$! ) { $pm->start(sub{ $sb->update(.); while(1) { my $job = $queue->dequeue; $sb->update(A); #work work work $sb->update(.); } });}

use Parallel::Prefork;use Parallel::Scoreboard;my $sb = Parallel::Scoreboard->new( base_dir => ‘/var/run/example_worker’ );my $pm = Parallel::Prefork->new({...});while ( $pm->signal_received !~ m!^(?:TERM|INT)$! ) { $pm->start(sub{ idle $sb->update(.); while(1) { my $job = $queue->dequeue; $sb->update(A); #work work work busy $sb->update(.); } }); idle}

# scoreboard.psgimy $sb = Parallel::Scoreboard->new( base_dir => ‘/var/run/example_worker’ );sub { my $env = shift; my $stats = $scoreboard->read_all(); my $busy = 0; my $idle = 0; for my $pid ( sort { $a <=> $b } keys %$stats) { if ( $stats->{$pid} =~ m!^A! ) { $busy++; } else { busy $idle++; } } return [200,[Content-Type=>plain/text], ["BusyWorkers: $busynIdleWorkers: $idle"]];};

#$ plackup -p 5001 -a scoreboard.psgi#$ curl http://localhost:5001/___server-statusBusyWorkers: 6IdleWorkers: 4

192.188.67.2 idle busy

http://blog.nomadscafe.jp	1506
http://yapcasia.org	85
http://paper.li	14
http://infra.rrdtool.net	13
http://blog.yapcasia.org	4
http://a0.twimg.com	3
http://us-w1.rockmelt.com	3
http://feeds.feedburner.jp	2
http://www.freerss.net	1
http://freerss.net	1
http://translate.googleusercontent.com	1

Designing Opeation Oriented Web Applications / YAPC::Asia Tokyo 2011

by Masahiro Nagano on Oct 14, 2011

Accessibility

Categories

Tags

Upload Details

Usage Rights

11 Embeds 1,633

Statistics

Designing Opeation Oriented Web Applications / YAPC::Asia Tokyo 2011 — Presentation Transcript