Introduction to CloudForecast
YAPC::Asia Tokyo 2010
                        @kazeburo
CloudForecast
P   D

A   C
P   D

A   C
CloudForecast
# ubuntu
$ sudo apt-get install librrds-perl libsnmp-perl

# CentOS (rrdtool EPEL             )
$ sudo rpm -Uvh http://dow...
$ git clone git://github.com/kazeburo/cloudforecast.git
$ cd cloudforecast
$ cpanm -L extlib --installdeps .
#
$ cp cloudforecast_sample.yaml cloudforecast.yaml

#
$ cp server_list_sample.yaml server_list.yaml
--- #Dev
servers:
  - config: basic.yaml
    hosts:
     - 127.0.0.1 server1

--- #Production
servers:
  - config: basic.yam...
--- #Dev
servers:
  - config: basic.yaml
    hosts:
     - 127.0.0.1 server1

--- #Production
servers:
  - config: basic.yam...
--- #Dev
servers:
  - config: basic.yaml
    hosts:
     - 127.0.0.1 server1

--- #Production
servers:
  - config: basic.yam...
--- #Dev
servers:
  - config: basic.yaml
    hosts:
     - 127.0.0.1 server1

--- #Production
servers:
  - config: basic.yam...
--- #Dev
servers:
  - config: basic.yaml
    hosts:
     - 127.0.0.1 server1

--- #Production
servers:
  - config: basic.yam...
--- #Dev
servers:
  - config: basic.yaml
    hosts:
     - 127.0.0.1 server1

--- #Production
servers:
  - config: basic.yam...
--- #Dev
servers:
  - config: basic.yaml
    hosts:
     - 127.0.0.1 server1

--- #Production
servers:
  - config: basic.yam...
--- #Dev
servers:
  - config: basic.yaml
    hosts:
     - 127.0.0.1 server1
     - 192.168.67.2 server2
--- #Production
se...
---
component_config:
resources:
  - basic
  - traffic:eth0
---
component_config:
resources:
  - basic
  - traffic:eth0
---
component_config:
resources:
  - basic
  - traffic:eth0
$ vim host_config/http8080_memcached11211.yaml


---
component_config:
resources:
  - traffic:eth1
  - basic
$ vim host_config/http8080_memcached11211.yaml


---
component_config:
resources:
  - traffic:eth1
  - basic
  - httpd:8080:/...
--- #MYHOME
servers:
  - config: basic.yaml
    hosts:
     - 127.0.0.1 server1
    - 192.168.67.2 server2
 - config: http80...
#
$ ./cloudforecast_radar -c cloudforecast.yaml 
                 -l server_list.yaml


# Web
$ ./cloudforecast_web -p 500...
# site-lib
$ mkdir -p site-lib/CloudForecast/Data
$ telnet localhost 7003
Trying 127.0.0.1...
Connected to localhost.
Escape character is '^]'.
status
fetcher 10       3   ...
$ telnet localhost 7003
Trying 127.0.0.1...
Connected to localhost.
Escape character is '^]'.
status
fetcher 10       3   ...
$ telnet localhost 7003
Trying 127.0.0.1...
Connected to localhost.
Escape character is '^]'.
status
fetcher 10       3   ...
$ telnet localhost 7003
Trying 127.0.0.1...
Connected to localhost.
Escape character is '^]'.
status
fetcher 10       3   ...
$ telnet localhost 7003
Trying 127.0.0.1...
Connected to localhost.
Escape character is '^]'.
status
fetcher 10       3   ...
$ telnet localhost 7003
Trying 127.0.0.1...
Connected to localhost.
Escape character is '^]'.
status
fetcher 10       3   ...
1: package CloudForecast::Data::Gearmand;
2:
3: use CloudForecast::Data -base;
4: use IO::Socket::INET;
5:
6: rrds ‘queue’...
1: package CloudForecast::Data::Gearmand;
2:
3: use CloudForecast::Data -base;
4: use IO::Socket::INET;
5:
6: rrds ‘queue’...
1: package CloudForecast::Data::Gearmand;
2:
3: use CloudForecast::Data -base;
4: use IO::Socket::INET;
5:
6: rrds ‘queue’...
1: package CloudForecast::Data::Gearmand;
2:
3: use CloudForecast::Data -base;
4: use IO::Socket::INET;
5:
6: rrds ‘queue’...
rrds ‘queue’, ‘GAUGE’;
# rrds ‘key’, ‘TYPE’;
# rrds [‘key’,‘TYPE’], []..;
graphs 'queue' => 'Queue';

# graphs ‘key1’ => ‘label’;
# graphs ‘key2’ => ‘the great graph’;
#
graphs 'queue' => 'Queue';

__DATA__
@@ queue
DEF:my1a=<%RRD%>:queue:AVERAGE
DEF:my2=<%RRD%>:proc:AVERAGE
CDEF:my1=my1a,my...
fetcher {                                                  my ($queue, $proc) = ( 0, 0 );
   my $c = shift;               ...
fetcher {                                                  my ($queue, $proc) = ( 0, 0 );
   my $c = shift;               ...
fetcher {                                                  my ($queue, $proc) = ( 0, 0 );
   my $c = shift;               ...
#                                            #
title {                                      sysinfo {
    my $c = shift;  ...
---
component_config:
resources:
  - traffic:eth1
  - basic
  - gearmand:7003
Tips
$ vim cloudforecast.yaml

---
config:
   gearman_enable: 1
   gearman_server:
    host: localhost
    port: 7003
   data_di...
#
$ ./cf_fetcher_worker -c cloudforecast.yaml 
                      --max-workers 20 
                      --max-executi...
#!/bin/sh

exec 2>&1
export CF_DEBUG=1
exec /path/to/cloudforecast/cloudforecast_radar
  -r 
  -c /path/to/cloudforecast/c...
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Introduction to CloudForecast / YAPC::Asia 2010 Tokyo
Upcoming SlideShare
Loading in...5
×

Introduction to CloudForecast / YAPC::Asia 2010 Tokyo

13,480

Published on

0 Comments
12 Likes
Statistics
Notes
  • Be the first to comment

No Downloads
Views
Total Views
13,480
On Slideshare
0
From Embeds
0
Number of Embeds
3
Actions
Shares
0
Downloads
76
Comments
0
Likes
12
Embeds 0
No embeds

No notes for slide














































































































































  • Introduction to CloudForecast / YAPC::Asia 2010 Tokyo

    1. 1. Introduction to CloudForecast YAPC::Asia Tokyo 2010 @kazeburo
    2. 2. CloudForecast
    3. 3. P D A C
    4. 4. P D A C
    5. 5. CloudForecast
    6. 6. # ubuntu $ sudo apt-get install librrds-perl libsnmp-perl # CentOS (rrdtool EPEL ) $ sudo rpm -Uvh http://download.fedora.redhat.com/ pub/epel/5/x86_64/epel-release-5-4.noarch.rpm $ sudo yum install net-snmp-perl $ sudo yum install rrdtool-perl
    7. 7. $ git clone git://github.com/kazeburo/cloudforecast.git $ cd cloudforecast $ cpanm -L extlib --installdeps .
    8. 8. # $ cp cloudforecast_sample.yaml cloudforecast.yaml # $ cp server_list_sample.yaml server_list.yaml
    9. 9. --- #Dev servers: - config: basic.yaml hosts: - 127.0.0.1 server1 --- #Production servers: - config: basic.yaml hosts: ....
    10. 10. --- #Dev servers: - config: basic.yaml hosts: - 127.0.0.1 server1 --- #Production servers: - config: basic.yaml hosts: ....
    11. 11. --- #Dev servers: - config: basic.yaml hosts: - 127.0.0.1 server1 --- #Production servers: - config: basic.yaml hosts: ....
    12. 12. --- #Dev servers: - config: basic.yaml hosts: - 127.0.0.1 server1 --- #Production servers: - config: basic.yaml hosts: ....
    13. 13. --- #Dev servers: - config: basic.yaml hosts: - 127.0.0.1 server1 --- #Production servers: - config: basic.yaml hosts: ....
    14. 14. --- #Dev servers: - config: basic.yaml hosts: - 127.0.0.1 server1 --- #Production servers: - config: basic.yaml hosts: ....
    15. 15. --- #Dev servers: - config: basic.yaml hosts: - 127.0.0.1 server1 --- #Production servers: - config: basic.yaml hosts: ....
    16. 16. --- #Dev servers: - config: basic.yaml hosts: - 127.0.0.1 server1 - 192.168.67.2 server2 --- #Production servers: - config: basic.yaml hosts: ....
    17. 17. --- component_config: resources: - basic - traffic:eth0
    18. 18. --- component_config: resources: - basic - traffic:eth0
    19. 19. --- component_config: resources: - basic - traffic:eth0
    20. 20. $ vim host_config/http8080_memcached11211.yaml --- component_config: resources: - traffic:eth1 - basic
    21. 21. $ vim host_config/http8080_memcached11211.yaml --- component_config: resources: - traffic:eth1 - basic - httpd:8080:/server-status?auto - memcached:11211
    22. 22. --- #MYHOME servers: - config: basic.yaml hosts: - 127.0.0.1 server1 - 192.168.67.2 server2 - config: http8080_memcached11211.yaml label: App hosts: - 192.168.67.10 server3 Web
    23. 23. # $ ./cloudforecast_radar -c cloudforecast.yaml -l server_list.yaml # Web $ ./cloudforecast_web -p 5000 -c cloudforecast.yaml -l server_list.yaml
    24. 24. # site-lib $ mkdir -p site-lib/CloudForecast/Data
    25. 25. $ telnet localhost 7003 Trying 127.0.0.1... Connected to localhost. Escape character is '^]'. status fetcher 10 3 8 updater 4 4 2 ledge 2 2 2 .
    26. 26. $ telnet localhost 7003 Trying 127.0.0.1... Connected to localhost. Escape character is '^]'. status fetcher 10 3 8 updater 4 4 2 ledge 2 2 2 .
    27. 27. $ telnet localhost 7003 Trying 127.0.0.1... Connected to localhost. Escape character is '^]'. status fetcher 10 3 8 updater 4 4 2 ledge 2 2 2 .
    28. 28. $ telnet localhost 7003 Trying 127.0.0.1... Connected to localhost. Escape character is '^]'. status fetcher 10 3 8 updater 4 4 2 ledge 2 2 2 .
    29. 29. $ telnet localhost 7003 Trying 127.0.0.1... Connected to localhost. Escape character is '^]'. status fetcher 10 3 8 updater 4 4 2 ledge 2 2 2 .
    30. 30. $ telnet localhost 7003 Trying 127.0.0.1... Connected to localhost. Escape character is '^]'. status fetcher 10 3 8 updater 4 4 2 ledge 2 2 2 .
    31. 31. 1: package CloudForecast::Data::Gearmand; 2: 3: use CloudForecast::Data -base; 4: use IO::Socket::INET; 5: 6: rrds ‘queue’, ‘GAUGE’; 7: rrds ‘proc’, ‘GAUGE’; 8: graphs 'queue' => 'Queue'; 9: ( )
    32. 32. 1: package CloudForecast::Data::Gearmand; 2: 3: use CloudForecast::Data -base; 4: use IO::Socket::INET; 5: 6: rrds ‘queue’, ‘GAUGE’; 7: rrds ‘proc’, ‘GAUGE’; 8: graphs 'queue' => 'Queue'; 9: ( )
    33. 33. 1: package CloudForecast::Data::Gearmand; 2: 3: use CloudForecast::Data -base; 4: use IO::Socket::INET; 5: 6: rrds ‘queue’, ‘GAUGE’; 7: rrds ‘proc’, ‘GAUGE’; 8: graphs 'queue' => 'Queue'; 9: ( )
    34. 34. 1: package CloudForecast::Data::Gearmand; 2: 3: use CloudForecast::Data -base; 4: use IO::Socket::INET; 5: 6: rrds ‘queue’, ‘GAUGE’; 7: rrds ‘proc’, ‘GAUGE’; 8: graphs 'queue' => 'Queue'; 9: ( )
    35. 35. rrds ‘queue’, ‘GAUGE’; # rrds ‘key’, ‘TYPE’; # rrds [‘key’,‘TYPE’], []..;
    36. 36. graphs 'queue' => 'Queue'; # graphs ‘key1’ => ‘label’; # graphs ‘key2’ => ‘the great graph’; #
    37. 37. graphs 'queue' => 'Queue'; __DATA__ @@ queue DEF:my1a=<%RRD%>:queue:AVERAGE DEF:my2=<%RRD%>:proc:AVERAGE CDEF:my1=my1a,my2,- AREA:my2#0000C0:Running GPRINT:my2:LAST:Cur: %6.1lf GPRINT:my2:AVERAGE:Ave: %6.1lf GPRINT:my2:MAX:Max: %6.1lf GPRINT:my2:MIN:Min: %6.1lfc STACK:my1#00C000:Queue GPRINT:my1:LAST:Cur: %6.1lf GPRINT:my1:AVERAGE:Ave: %6.1lf GPRINT:my1:MAX:Max: %6.1lf GPRINT:my1:MIN:Min: %6.1lfc
    38. 38. fetcher { my ($queue, $proc) = ( 0, 0 ); my $c = shift; my @sysinfo = qw/job worker/; for my $job ( values %status ) { my $host = $c->address; $queue += $job->[1]; my $port = $c->args->[0] || 7003; $proc += $job->[2]; } my $sock = IO::Socket::INET->new( PeerAddr => $host, return [ $queue, $proc ]; PeerPort => $port, }; ); $sock->syswrite("statusrn"); $sock->sysread( my $raw_status, 8192 ); my %status; foreach my $line ( split /r?n/, $raw_status ) { my @st = split /s+/, $line; next unless @st == 4; $status{$st[0]} = @st; }
    39. 39. fetcher { my ($queue, $proc) = ( 0, 0 ); my $c = shift; my @sysinfo = qw/job worker/; for my $job ( values %status ) { my $host = $c->address; $queue += $job->[1]; my $port = $c->args->[0] || 7003; $proc += $job->[2]; } my $sock = IO::Socket::INET->new( PeerAddr => $host, return [ $queue, $proc ]; PeerPort => $port, }; ); $sock->syswrite("statusrn"); $sock->sysread( my $raw_status, 8192 ); my %status; foreach my $line ( split /r?n/, $raw_status ) { my @st = split /s+/, $line; next unless @st == 4; $status{$st[0]} = @st; }
    40. 40. fetcher { my ($queue, $proc) = ( 0, 0 ); my $c = shift; my @sysinfo = qw/job worker/; for my $job ( values %status ) { my $host = $c->address; $queue += $job->[1]; my $port = $c->args->[0] || 7003; $proc += $job->[2]; } my $sock = IO::Socket::INET->new( PeerAddr => $host, return [ $queue, $proc ]; PeerPort => $port, }; ); $sock->syswrite("statusrn"); $sock->sysread( my $raw_status, 8192 ); my %status; foreach my $line ( split /r?n/, $raw_status ) { my @st = split /s+/, $line; next unless @st == 4; $status{$st[0]} = @st; }
    41. 41. # # title { sysinfo { my $c = shift; my $c = shift; my $title = "Gearmand"; #$c->ledge_get('sysinfo') || []; if ( my $port = $c->args->[0] ) { [‘job’,‘worker’]; $title .= " ($port)"; }; } return $title; }; # KVS $c->ledge_set( 'key', @data, [expires] ); $c->ledge_add( 'key', @data, [expires] ); $c->ledge_get( 'key’ ); $c->ledge_delete( 'key’ );
    42. 42. --- component_config: resources: - traffic:eth1 - basic - gearmand:7003
    43. 43. Tips
    44. 44. $ vim cloudforecast.yaml --- config: gearman_enable: 1 gearman_server: host: localhost port: 7003 data_dir: data host_config_dir: host_config ..
    45. 45. # $ ./cf_fetcher_worker -c cloudforecast.yaml --max-workers 20 --max-execution-time 60 # RRD $ ./cf_updater_worker -c cloudforecast.yaml --max-workers 4
    46. 46. #!/bin/sh exec 2>&1 export CF_DEBUG=1 exec /path/to/cloudforecast/cloudforecast_radar -r -c /path/to/cloudforecast/cloudforecast.yaml -l /path/to/cloudforecast/server_list.yaml
    1. A particular slide catching your eye?

      Clipping is a handy way to collect important slides you want to go back to later.

    ×