Introduction to CloudForecast / YAPC::Asia 2010 Tokyo

  • 12,778 views
Uploaded on

 

  • Full Name Full Name Comment goes here.
    Are you sure you want to
    Your message goes here
    Be the first to comment
No Downloads

Views

Total Views
12,778
On Slideshare
0
From Embeds
0
Number of Embeds
2

Actions

Shares
Downloads
76
Comments
0
Likes
11

Embeds 0

No embeds

Report content

Flagged as inappropriate Flag as inappropriate
Flag as inappropriate

Select your reason for flagging this presentation as inappropriate.

Cancel
    No notes for slide














































































































































Transcript

  • 1. Introduction to CloudForecast YAPC::Asia Tokyo 2010 @kazeburo
  • 2. CloudForecast
  • 3. P D A C
  • 4. P D A C
  • 5. CloudForecast
  • 6. # ubuntu $ sudo apt-get install librrds-perl libsnmp-perl # CentOS (rrdtool EPEL ) $ sudo rpm -Uvh http://download.fedora.redhat.com/ pub/epel/5/x86_64/epel-release-5-4.noarch.rpm $ sudo yum install net-snmp-perl $ sudo yum install rrdtool-perl
  • 7. $ git clone git://github.com/kazeburo/cloudforecast.git $ cd cloudforecast $ cpanm -L extlib --installdeps .
  • 8. # $ cp cloudforecast_sample.yaml cloudforecast.yaml # $ cp server_list_sample.yaml server_list.yaml
  • 9. --- #Dev servers: - config: basic.yaml hosts: - 127.0.0.1 server1 --- #Production servers: - config: basic.yaml hosts: ....
  • 10. --- #Dev servers: - config: basic.yaml hosts: - 127.0.0.1 server1 --- #Production servers: - config: basic.yaml hosts: ....
  • 11. --- #Dev servers: - config: basic.yaml hosts: - 127.0.0.1 server1 --- #Production servers: - config: basic.yaml hosts: ....
  • 12. --- #Dev servers: - config: basic.yaml hosts: - 127.0.0.1 server1 --- #Production servers: - config: basic.yaml hosts: ....
  • 13. --- #Dev servers: - config: basic.yaml hosts: - 127.0.0.1 server1 --- #Production servers: - config: basic.yaml hosts: ....
  • 14. --- #Dev servers: - config: basic.yaml hosts: - 127.0.0.1 server1 --- #Production servers: - config: basic.yaml hosts: ....
  • 15. --- #Dev servers: - config: basic.yaml hosts: - 127.0.0.1 server1 --- #Production servers: - config: basic.yaml hosts: ....
  • 16. --- #Dev servers: - config: basic.yaml hosts: - 127.0.0.1 server1 - 192.168.67.2 server2 --- #Production servers: - config: basic.yaml hosts: ....
  • 17. --- component_config: resources: - basic - traffic:eth0
  • 18. --- component_config: resources: - basic - traffic:eth0
  • 19. --- component_config: resources: - basic - traffic:eth0
  • 20. $ vim host_config/http8080_memcached11211.yaml --- component_config: resources: - traffic:eth1 - basic
  • 21. $ vim host_config/http8080_memcached11211.yaml --- component_config: resources: - traffic:eth1 - basic - httpd:8080:/server-status?auto - memcached:11211
  • 22. --- #MYHOME servers: - config: basic.yaml hosts: - 127.0.0.1 server1 - 192.168.67.2 server2 - config: http8080_memcached11211.yaml label: App hosts: - 192.168.67.10 server3 Web
  • 23. # $ ./cloudforecast_radar -c cloudforecast.yaml -l server_list.yaml # Web $ ./cloudforecast_web -p 5000 -c cloudforecast.yaml -l server_list.yaml
  • 24. # site-lib $ mkdir -p site-lib/CloudForecast/Data
  • 25. $ telnet localhost 7003 Trying 127.0.0.1... Connected to localhost. Escape character is '^]'. status fetcher 10 3 8 updater 4 4 2 ledge 2 2 2 .
  • 26. $ telnet localhost 7003 Trying 127.0.0.1... Connected to localhost. Escape character is '^]'. status fetcher 10 3 8 updater 4 4 2 ledge 2 2 2 .
  • 27. $ telnet localhost 7003 Trying 127.0.0.1... Connected to localhost. Escape character is '^]'. status fetcher 10 3 8 updater 4 4 2 ledge 2 2 2 .
  • 28. $ telnet localhost 7003 Trying 127.0.0.1... Connected to localhost. Escape character is '^]'. status fetcher 10 3 8 updater 4 4 2 ledge 2 2 2 .
  • 29. $ telnet localhost 7003 Trying 127.0.0.1... Connected to localhost. Escape character is '^]'. status fetcher 10 3 8 updater 4 4 2 ledge 2 2 2 .
  • 30. $ telnet localhost 7003 Trying 127.0.0.1... Connected to localhost. Escape character is '^]'. status fetcher 10 3 8 updater 4 4 2 ledge 2 2 2 .
  • 31. 1: package CloudForecast::Data::Gearmand; 2: 3: use CloudForecast::Data -base; 4: use IO::Socket::INET; 5: 6: rrds ‘queue’, ‘GAUGE’; 7: rrds ‘proc’, ‘GAUGE’; 8: graphs 'queue' => 'Queue'; 9: ( )
  • 32. 1: package CloudForecast::Data::Gearmand; 2: 3: use CloudForecast::Data -base; 4: use IO::Socket::INET; 5: 6: rrds ‘queue’, ‘GAUGE’; 7: rrds ‘proc’, ‘GAUGE’; 8: graphs 'queue' => 'Queue'; 9: ( )
  • 33. 1: package CloudForecast::Data::Gearmand; 2: 3: use CloudForecast::Data -base; 4: use IO::Socket::INET; 5: 6: rrds ‘queue’, ‘GAUGE’; 7: rrds ‘proc’, ‘GAUGE’; 8: graphs 'queue' => 'Queue'; 9: ( )
  • 34. 1: package CloudForecast::Data::Gearmand; 2: 3: use CloudForecast::Data -base; 4: use IO::Socket::INET; 5: 6: rrds ‘queue’, ‘GAUGE’; 7: rrds ‘proc’, ‘GAUGE’; 8: graphs 'queue' => 'Queue'; 9: ( )
  • 35. rrds ‘queue’, ‘GAUGE’; # rrds ‘key’, ‘TYPE’; # rrds [‘key’,‘TYPE’], []..;
  • 36. graphs 'queue' => 'Queue'; # graphs ‘key1’ => ‘label’; # graphs ‘key2’ => ‘the great graph’; #
  • 37. graphs 'queue' => 'Queue'; __DATA__ @@ queue DEF:my1a=<%RRD%>:queue:AVERAGE DEF:my2=<%RRD%>:proc:AVERAGE CDEF:my1=my1a,my2,- AREA:my2#0000C0:Running GPRINT:my2:LAST:Cur: %6.1lf GPRINT:my2:AVERAGE:Ave: %6.1lf GPRINT:my2:MAX:Max: %6.1lf GPRINT:my2:MIN:Min: %6.1lfc STACK:my1#00C000:Queue GPRINT:my1:LAST:Cur: %6.1lf GPRINT:my1:AVERAGE:Ave: %6.1lf GPRINT:my1:MAX:Max: %6.1lf GPRINT:my1:MIN:Min: %6.1lfc
  • 38. fetcher { my ($queue, $proc) = ( 0, 0 ); my $c = shift; my @sysinfo = qw/job worker/; for my $job ( values %status ) { my $host = $c->address; $queue += $job->[1]; my $port = $c->args->[0] || 7003; $proc += $job->[2]; } my $sock = IO::Socket::INET->new( PeerAddr => $host, return [ $queue, $proc ]; PeerPort => $port, }; ); $sock->syswrite("statusrn"); $sock->sysread( my $raw_status, 8192 ); my %status; foreach my $line ( split /r?n/, $raw_status ) { my @st = split /s+/, $line; next unless @st == 4; $status{$st[0]} = @st; }
  • 39. fetcher { my ($queue, $proc) = ( 0, 0 ); my $c = shift; my @sysinfo = qw/job worker/; for my $job ( values %status ) { my $host = $c->address; $queue += $job->[1]; my $port = $c->args->[0] || 7003; $proc += $job->[2]; } my $sock = IO::Socket::INET->new( PeerAddr => $host, return [ $queue, $proc ]; PeerPort => $port, }; ); $sock->syswrite("statusrn"); $sock->sysread( my $raw_status, 8192 ); my %status; foreach my $line ( split /r?n/, $raw_status ) { my @st = split /s+/, $line; next unless @st == 4; $status{$st[0]} = @st; }
  • 40. fetcher { my ($queue, $proc) = ( 0, 0 ); my $c = shift; my @sysinfo = qw/job worker/; for my $job ( values %status ) { my $host = $c->address; $queue += $job->[1]; my $port = $c->args->[0] || 7003; $proc += $job->[2]; } my $sock = IO::Socket::INET->new( PeerAddr => $host, return [ $queue, $proc ]; PeerPort => $port, }; ); $sock->syswrite("statusrn"); $sock->sysread( my $raw_status, 8192 ); my %status; foreach my $line ( split /r?n/, $raw_status ) { my @st = split /s+/, $line; next unless @st == 4; $status{$st[0]} = @st; }
  • 41. # # title { sysinfo { my $c = shift; my $c = shift; my $title = "Gearmand"; #$c->ledge_get('sysinfo') || []; if ( my $port = $c->args->[0] ) { [‘job’,‘worker’]; $title .= " ($port)"; }; } return $title; }; # KVS $c->ledge_set( 'key', @data, [expires] ); $c->ledge_add( 'key', @data, [expires] ); $c->ledge_get( 'key’ ); $c->ledge_delete( 'key’ );
  • 42. --- component_config: resources: - traffic:eth1 - basic - gearmand:7003
  • 43. Tips
  • 44. $ vim cloudforecast.yaml --- config: gearman_enable: 1 gearman_server: host: localhost port: 7003 data_dir: data host_config_dir: host_config ..
  • 45. # $ ./cf_fetcher_worker -c cloudforecast.yaml --max-workers 20 --max-execution-time 60 # RRD $ ./cf_updater_worker -c cloudforecast.yaml --max-workers 4
  • 46. #!/bin/sh exec 2>&1 export CF_DEBUG=1 exec /path/to/cloudforecast/cloudforecast_radar -r -c /path/to/cloudforecast/cloudforecast.yaml -l /path/to/cloudforecast/server_list.yaml