Successfully reported this slideshow.
We use your LinkedIn profile and activity data to personalize ads and to show you more relevant ads. You can change your ad preferences anytime.
Introduction to CloudForecast
YAPC::Asia Tokyo 2010
                        @kazeburo
CloudForecast
P   D

A   C
P   D

A   C
CloudForecast
# ubuntu
$ sudo apt-get install librrds-perl libsnmp-perl

# CentOS (rrdtool EPEL             )
$ sudo rpm -Uvh http://dow...
$ git clone git://github.com/kazeburo/cloudforecast.git
$ cd cloudforecast
$ cpanm -L extlib --installdeps .
#
$ cp cloudforecast_sample.yaml cloudforecast.yaml

#
$ cp server_list_sample.yaml server_list.yaml
--- #Dev
servers:
  - config: basic.yaml
    hosts:
     - 127.0.0.1 server1

--- #Production
servers:
  - config: basic.yam...
--- #Dev
servers:
  - config: basic.yaml
    hosts:
     - 127.0.0.1 server1

--- #Production
servers:
  - config: basic.yam...
--- #Dev
servers:
  - config: basic.yaml
    hosts:
     - 127.0.0.1 server1

--- #Production
servers:
  - config: basic.yam...
--- #Dev
servers:
  - config: basic.yaml
    hosts:
     - 127.0.0.1 server1

--- #Production
servers:
  - config: basic.yam...
--- #Dev
servers:
  - config: basic.yaml
    hosts:
     - 127.0.0.1 server1

--- #Production
servers:
  - config: basic.yam...
--- #Dev
servers:
  - config: basic.yaml
    hosts:
     - 127.0.0.1 server1

--- #Production
servers:
  - config: basic.yam...
--- #Dev
servers:
  - config: basic.yaml
    hosts:
     - 127.0.0.1 server1

--- #Production
servers:
  - config: basic.yam...
--- #Dev
servers:
  - config: basic.yaml
    hosts:
     - 127.0.0.1 server1
     - 192.168.67.2 server2
--- #Production
se...
---
component_config:
resources:
  - basic
  - traffic:eth0
---
component_config:
resources:
  - basic
  - traffic:eth0
---
component_config:
resources:
  - basic
  - traffic:eth0
$ vim host_config/http8080_memcached11211.yaml


---
component_config:
resources:
  - traffic:eth1
  - basic
$ vim host_config/http8080_memcached11211.yaml


---
component_config:
resources:
  - traffic:eth1
  - basic
  - httpd:8080:/...
--- #MYHOME
servers:
  - config: basic.yaml
    hosts:
     - 127.0.0.1 server1
    - 192.168.67.2 server2
 - config: http80...
#
$ ./cloudforecast_radar -c cloudforecast.yaml 
                 -l server_list.yaml


# Web
$ ./cloudforecast_web -p 500...
# site-lib
$ mkdir -p site-lib/CloudForecast/Data
$ telnet localhost 7003
Trying 127.0.0.1...
Connected to localhost.
Escape character is '^]'.
status
fetcher 10       3   ...
$ telnet localhost 7003
Trying 127.0.0.1...
Connected to localhost.
Escape character is '^]'.
status
fetcher 10       3   ...
$ telnet localhost 7003
Trying 127.0.0.1...
Connected to localhost.
Escape character is '^]'.
status
fetcher 10       3   ...
$ telnet localhost 7003
Trying 127.0.0.1...
Connected to localhost.
Escape character is '^]'.
status
fetcher 10       3   ...
$ telnet localhost 7003
Trying 127.0.0.1...
Connected to localhost.
Escape character is '^]'.
status
fetcher 10       3   ...
$ telnet localhost 7003
Trying 127.0.0.1...
Connected to localhost.
Escape character is '^]'.
status
fetcher 10       3   ...
1: package CloudForecast::Data::Gearmand;
2:
3: use CloudForecast::Data -base;
4: use IO::Socket::INET;
5:
6: rrds ‘queue’...
1: package CloudForecast::Data::Gearmand;
2:
3: use CloudForecast::Data -base;
4: use IO::Socket::INET;
5:
6: rrds ‘queue’...
1: package CloudForecast::Data::Gearmand;
2:
3: use CloudForecast::Data -base;
4: use IO::Socket::INET;
5:
6: rrds ‘queue’...
1: package CloudForecast::Data::Gearmand;
2:
3: use CloudForecast::Data -base;
4: use IO::Socket::INET;
5:
6: rrds ‘queue’...
rrds ‘queue’, ‘GAUGE’;
# rrds ‘key’, ‘TYPE’;
# rrds [‘key’,‘TYPE’], []..;
graphs 'queue' => 'Queue';

# graphs ‘key1’ => ‘label’;
# graphs ‘key2’ => ‘the great graph’;
#
graphs 'queue' => 'Queue';

__DATA__
@@ queue
DEF:my1a=<%RRD%>:queue:AVERAGE
DEF:my2=<%RRD%>:proc:AVERAGE
CDEF:my1=my1a,my...
fetcher {                                                  my ($queue, $proc) = ( 0, 0 );
   my $c = shift;               ...
fetcher {                                                  my ($queue, $proc) = ( 0, 0 );
   my $c = shift;               ...
fetcher {                                                  my ($queue, $proc) = ( 0, 0 );
   my $c = shift;               ...
#                                            #
title {                                      sysinfo {
    my $c = shift;  ...
---
component_config:
resources:
  - traffic:eth1
  - basic
  - gearmand:7003
Tips
$ vim cloudforecast.yaml

---
config:
   gearman_enable: 1
   gearman_server:
    host: localhost
    port: 7003
   data_di...
#
$ ./cf_fetcher_worker -c cloudforecast.yaml 
                      --max-workers 20 
                      --max-executi...
#!/bin/sh

exec 2>&1
export CF_DEBUG=1
exec /path/to/cloudforecast/cloudforecast_radar
  -r 
  -c /path/to/cloudforecast/c...
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Upcoming SlideShare
Loading in …5
×

Introduction to CloudForecast / YAPC::Asia 2010 Tokyo

18,592 views

Published on

  • Be the first to comment

Introduction to CloudForecast / YAPC::Asia 2010 Tokyo

  1. Introduction to CloudForecast YAPC::Asia Tokyo 2010 @kazeburo
  2. CloudForecast
  3. P D A C
  4. P D A C
  5. CloudForecast
  6. # ubuntu $ sudo apt-get install librrds-perl libsnmp-perl # CentOS (rrdtool EPEL ) $ sudo rpm -Uvh http://download.fedora.redhat.com/ pub/epel/5/x86_64/epel-release-5-4.noarch.rpm $ sudo yum install net-snmp-perl $ sudo yum install rrdtool-perl
  7. $ git clone git://github.com/kazeburo/cloudforecast.git $ cd cloudforecast $ cpanm -L extlib --installdeps .
  8. # $ cp cloudforecast_sample.yaml cloudforecast.yaml # $ cp server_list_sample.yaml server_list.yaml
  9. --- #Dev servers: - config: basic.yaml hosts: - 127.0.0.1 server1 --- #Production servers: - config: basic.yaml hosts: ....
  10. --- #Dev servers: - config: basic.yaml hosts: - 127.0.0.1 server1 --- #Production servers: - config: basic.yaml hosts: ....
  11. --- #Dev servers: - config: basic.yaml hosts: - 127.0.0.1 server1 --- #Production servers: - config: basic.yaml hosts: ....
  12. --- #Dev servers: - config: basic.yaml hosts: - 127.0.0.1 server1 --- #Production servers: - config: basic.yaml hosts: ....
  13. --- #Dev servers: - config: basic.yaml hosts: - 127.0.0.1 server1 --- #Production servers: - config: basic.yaml hosts: ....
  14. --- #Dev servers: - config: basic.yaml hosts: - 127.0.0.1 server1 --- #Production servers: - config: basic.yaml hosts: ....
  15. --- #Dev servers: - config: basic.yaml hosts: - 127.0.0.1 server1 --- #Production servers: - config: basic.yaml hosts: ....
  16. --- #Dev servers: - config: basic.yaml hosts: - 127.0.0.1 server1 - 192.168.67.2 server2 --- #Production servers: - config: basic.yaml hosts: ....
  17. --- component_config: resources: - basic - traffic:eth0
  18. --- component_config: resources: - basic - traffic:eth0
  19. --- component_config: resources: - basic - traffic:eth0
  20. $ vim host_config/http8080_memcached11211.yaml --- component_config: resources: - traffic:eth1 - basic
  21. $ vim host_config/http8080_memcached11211.yaml --- component_config: resources: - traffic:eth1 - basic - httpd:8080:/server-status?auto - memcached:11211
  22. --- #MYHOME servers: - config: basic.yaml hosts: - 127.0.0.1 server1 - 192.168.67.2 server2 - config: http8080_memcached11211.yaml label: App hosts: - 192.168.67.10 server3 Web
  23. # $ ./cloudforecast_radar -c cloudforecast.yaml -l server_list.yaml # Web $ ./cloudforecast_web -p 5000 -c cloudforecast.yaml -l server_list.yaml
  24. # site-lib $ mkdir -p site-lib/CloudForecast/Data
  25. $ telnet localhost 7003 Trying 127.0.0.1... Connected to localhost. Escape character is '^]'. status fetcher 10 3 8 updater 4 4 2 ledge 2 2 2 .
  26. $ telnet localhost 7003 Trying 127.0.0.1... Connected to localhost. Escape character is '^]'. status fetcher 10 3 8 updater 4 4 2 ledge 2 2 2 .
  27. $ telnet localhost 7003 Trying 127.0.0.1... Connected to localhost. Escape character is '^]'. status fetcher 10 3 8 updater 4 4 2 ledge 2 2 2 .
  28. $ telnet localhost 7003 Trying 127.0.0.1... Connected to localhost. Escape character is '^]'. status fetcher 10 3 8 updater 4 4 2 ledge 2 2 2 .
  29. $ telnet localhost 7003 Trying 127.0.0.1... Connected to localhost. Escape character is '^]'. status fetcher 10 3 8 updater 4 4 2 ledge 2 2 2 .
  30. $ telnet localhost 7003 Trying 127.0.0.1... Connected to localhost. Escape character is '^]'. status fetcher 10 3 8 updater 4 4 2 ledge 2 2 2 .
  31. 1: package CloudForecast::Data::Gearmand; 2: 3: use CloudForecast::Data -base; 4: use IO::Socket::INET; 5: 6: rrds ‘queue’, ‘GAUGE’; 7: rrds ‘proc’, ‘GAUGE’; 8: graphs 'queue' => 'Queue'; 9: ( )
  32. 1: package CloudForecast::Data::Gearmand; 2: 3: use CloudForecast::Data -base; 4: use IO::Socket::INET; 5: 6: rrds ‘queue’, ‘GAUGE’; 7: rrds ‘proc’, ‘GAUGE’; 8: graphs 'queue' => 'Queue'; 9: ( )
  33. 1: package CloudForecast::Data::Gearmand; 2: 3: use CloudForecast::Data -base; 4: use IO::Socket::INET; 5: 6: rrds ‘queue’, ‘GAUGE’; 7: rrds ‘proc’, ‘GAUGE’; 8: graphs 'queue' => 'Queue'; 9: ( )
  34. 1: package CloudForecast::Data::Gearmand; 2: 3: use CloudForecast::Data -base; 4: use IO::Socket::INET; 5: 6: rrds ‘queue’, ‘GAUGE’; 7: rrds ‘proc’, ‘GAUGE’; 8: graphs 'queue' => 'Queue'; 9: ( )
  35. rrds ‘queue’, ‘GAUGE’; # rrds ‘key’, ‘TYPE’; # rrds [‘key’,‘TYPE’], []..;
  36. graphs 'queue' => 'Queue'; # graphs ‘key1’ => ‘label’; # graphs ‘key2’ => ‘the great graph’; #
  37. graphs 'queue' => 'Queue'; __DATA__ @@ queue DEF:my1a=<%RRD%>:queue:AVERAGE DEF:my2=<%RRD%>:proc:AVERAGE CDEF:my1=my1a,my2,- AREA:my2#0000C0:Running GPRINT:my2:LAST:Cur: %6.1lf GPRINT:my2:AVERAGE:Ave: %6.1lf GPRINT:my2:MAX:Max: %6.1lf GPRINT:my2:MIN:Min: %6.1lfc STACK:my1#00C000:Queue GPRINT:my1:LAST:Cur: %6.1lf GPRINT:my1:AVERAGE:Ave: %6.1lf GPRINT:my1:MAX:Max: %6.1lf GPRINT:my1:MIN:Min: %6.1lfc
  38. fetcher { my ($queue, $proc) = ( 0, 0 ); my $c = shift; my @sysinfo = qw/job worker/; for my $job ( values %status ) { my $host = $c->address; $queue += $job->[1]; my $port = $c->args->[0] || 7003; $proc += $job->[2]; } my $sock = IO::Socket::INET->new( PeerAddr => $host, return [ $queue, $proc ]; PeerPort => $port, }; ); $sock->syswrite("statusrn"); $sock->sysread( my $raw_status, 8192 ); my %status; foreach my $line ( split /r?n/, $raw_status ) { my @st = split /s+/, $line; next unless @st == 4; $status{$st[0]} = @st; }
  39. fetcher { my ($queue, $proc) = ( 0, 0 ); my $c = shift; my @sysinfo = qw/job worker/; for my $job ( values %status ) { my $host = $c->address; $queue += $job->[1]; my $port = $c->args->[0] || 7003; $proc += $job->[2]; } my $sock = IO::Socket::INET->new( PeerAddr => $host, return [ $queue, $proc ]; PeerPort => $port, }; ); $sock->syswrite("statusrn"); $sock->sysread( my $raw_status, 8192 ); my %status; foreach my $line ( split /r?n/, $raw_status ) { my @st = split /s+/, $line; next unless @st == 4; $status{$st[0]} = @st; }
  40. fetcher { my ($queue, $proc) = ( 0, 0 ); my $c = shift; my @sysinfo = qw/job worker/; for my $job ( values %status ) { my $host = $c->address; $queue += $job->[1]; my $port = $c->args->[0] || 7003; $proc += $job->[2]; } my $sock = IO::Socket::INET->new( PeerAddr => $host, return [ $queue, $proc ]; PeerPort => $port, }; ); $sock->syswrite("statusrn"); $sock->sysread( my $raw_status, 8192 ); my %status; foreach my $line ( split /r?n/, $raw_status ) { my @st = split /s+/, $line; next unless @st == 4; $status{$st[0]} = @st; }
  41. # # title { sysinfo { my $c = shift; my $c = shift; my $title = "Gearmand"; #$c->ledge_get('sysinfo') || []; if ( my $port = $c->args->[0] ) { [‘job’,‘worker’]; $title .= " ($port)"; }; } return $title; }; # KVS $c->ledge_set( 'key', @data, [expires] ); $c->ledge_add( 'key', @data, [expires] ); $c->ledge_get( 'key’ ); $c->ledge_delete( 'key’ );
  42. --- component_config: resources: - traffic:eth1 - basic - gearmand:7003
  43. Tips
  44. $ vim cloudforecast.yaml --- config: gearman_enable: 1 gearman_server: host: localhost port: 7003 data_dir: data host_config_dir: host_config ..
  45. # $ ./cf_fetcher_worker -c cloudforecast.yaml --max-workers 20 --max-execution-time 60 # RRD $ ./cf_updater_worker -c cloudforecast.yaml --max-workers 4
  46. #!/bin/sh exec 2>&1 export CF_DEBUG=1 exec /path/to/cloudforecast/cloudforecast_radar -r -c /path/to/cloudforecast/cloudforecast.yaml -l /path/to/cloudforecast/server_list.yaml

×