1. Introduction to CloudForecast
YAPC::Asia Tokyo 2010
@kazeburo

4. CloudForecast

10. P D
A C

11. P D
A C

20. CloudForecast

33. # ubuntu
$ sudo apt-get install librrds-perl libsnmp-perl
# CentOS (rrdtool EPEL )
$ sudo rpm -Uvh http://download.fedora.redhat.com/
pub/epel/5/x86_64/epel-release-5-4.noarch.rpm
$ sudo yum install net-snmp-perl
$ sudo yum install rrdtool-perl

34. $ git clone git://github.com/kazeburo/cloudforecast.git
$ cd cloudforecast
$ cpanm -L extlib --installdeps .

35. #
$ cp cloudforecast_sample.yaml cloudforecast.yaml
#
$ cp server_list_sample.yaml server_list.yaml

36. --- #Dev
servers:
- config: basic.yaml
hosts:
- 127.0.0.1 server1
--- #Production
servers:
- config: basic.yaml
hosts:
....

37. --- #Dev
servers:
- config: basic.yaml
hosts:
- 127.0.0.1 server1
--- #Production
servers:
- config: basic.yaml
hosts:
....

38. --- #Dev
servers:
- config: basic.yaml
hosts:
- 127.0.0.1 server1
--- #Production
servers:
- config: basic.yaml
hosts:
....

39. --- #Dev
servers:
- config: basic.yaml
hosts:
- 127.0.0.1 server1
--- #Production
servers:
- config: basic.yaml
hosts:
....

40. --- #Dev
servers:
- config: basic.yaml
hosts:
- 127.0.0.1 server1
--- #Production
servers:
- config: basic.yaml
hosts:
....

41. --- #Dev
servers:
- config: basic.yaml
hosts:
- 127.0.0.1 server1
--- #Production
servers:
- config: basic.yaml
hosts:
....

42. --- #Dev
servers:
- config: basic.yaml
hosts:
- 127.0.0.1 server1
--- #Production
servers:
- config: basic.yaml
hosts:
....

43. --- #Dev
servers:
- config: basic.yaml
hosts:
- 127.0.0.1 server1
- 192.168.67.2 server2
--- #Production
servers:
- config: basic.yaml
hosts:
....

44. ---
component_config:
resources:
- basic
- traffic:eth0

45. ---
component_config:
resources:
- basic
- traffic:eth0

46. ---
component_config:
resources:
- basic
- traffic:eth0

49. $ vim host_config/http8080_memcached11211.yaml
---
component_config:
resources:
- traffic:eth1
- basic

50. $ vim host_config/http8080_memcached11211.yaml
---
component_config:
resources:
- traffic:eth1
- basic
- httpd:8080:/server-status?auto
- memcached:11211

51. --- #MYHOME
servers:
- config: basic.yaml
hosts:
- 127.0.0.1 server1
- 192.168.67.2 server2
- config: http8080_memcached11211.yaml
label: App
hosts:
- 192.168.67.10 server3 Web

52. #
$ ./cloudforecast_radar -c cloudforecast.yaml
-l server_list.yaml
# Web
$ ./cloudforecast_web -p 5000 -c cloudforecast.yaml
-l server_list.yaml

64. # site-lib
$ mkdir -p site-lib/CloudForecast/Data

66. $ telnet localhost 7003
Trying 127.0.0.1...
Connected to localhost.
Escape character is '^]'.
status
fetcher 10 3 8
updater 4 4 2
ledge 2 2 2
.

67. $ telnet localhost 7003
Trying 127.0.0.1...
Connected to localhost.
Escape character is '^]'.
status
fetcher 10 3 8
updater 4 4 2
ledge 2 2 2
.

68. $ telnet localhost 7003
Trying 127.0.0.1...
Connected to localhost.
Escape character is '^]'.
status
fetcher 10 3 8
updater 4 4 2
ledge 2 2 2
.

69. $ telnet localhost 7003
Trying 127.0.0.1...
Connected to localhost.
Escape character is '^]'.
status
fetcher 10 3 8
updater 4 4 2
ledge 2 2 2
.

70. $ telnet localhost 7003
Trying 127.0.0.1...
Connected to localhost.
Escape character is '^]'.
status
fetcher 10 3 8
updater 4 4 2
ledge 2 2 2
.

71. $ telnet localhost 7003
Trying 127.0.0.1...
Connected to localhost.
Escape character is '^]'.
status
fetcher 10 3 8
updater 4 4 2
ledge 2 2 2
.

72. 1: package CloudForecast::Data::Gearmand;
2:
3: use CloudForecast::Data -base;
4: use IO::Socket::INET;
5:
6: rrds ‘queue’, ‘GAUGE’;
7: rrds ‘proc’, ‘GAUGE’;
8: graphs 'queue' => 'Queue';
9:
( )

73. 1: package CloudForecast::Data::Gearmand;
2:
3: use CloudForecast::Data -base;
4: use IO::Socket::INET;
5:
6: rrds ‘queue’, ‘GAUGE’;
7: rrds ‘proc’, ‘GAUGE’;
8: graphs 'queue' => 'Queue';
9:
( )

74. 1: package CloudForecast::Data::Gearmand;
2:
3: use CloudForecast::Data -base;
4: use IO::Socket::INET;
5:
6: rrds ‘queue’, ‘GAUGE’;
7: rrds ‘proc’, ‘GAUGE’;
8: graphs 'queue' => 'Queue';
9:
( )

75. 1: package CloudForecast::Data::Gearmand;
2:
3: use CloudForecast::Data -base;
4: use IO::Socket::INET;
5:
6: rrds ‘queue’, ‘GAUGE’;
7: rrds ‘proc’, ‘GAUGE’;
8: graphs 'queue' => 'Queue';
9:
( )

76. rrds ‘queue’, ‘GAUGE’;
# rrds ‘key’, ‘TYPE’;
# rrds [‘key’,‘TYPE’], []..;

77. graphs 'queue' => 'Queue';
# graphs ‘key1’ => ‘label’;
# graphs ‘key2’ => ‘the great graph’;
#

78. graphs 'queue' => 'Queue';
__DATA__
@@ queue
DEF:my1a=<%RRD%>:queue:AVERAGE
DEF:my2=<%RRD%>:proc:AVERAGE
CDEF:my1=my1a,my2,-
AREA:my2#0000C0:Running
GPRINT:my2:LAST:Cur: %6.1lf
GPRINT:my2:AVERAGE:Ave: %6.1lf
GPRINT:my2:MAX:Max: %6.1lf
GPRINT:my2:MIN:Min: %6.1lfc
STACK:my1#00C000:Queue
GPRINT:my1:LAST:Cur: %6.1lf
GPRINT:my1:AVERAGE:Ave: %6.1lf
GPRINT:my1:MAX:Max: %6.1lf
GPRINT:my1:MIN:Min: %6.1lfc

79. fetcher { my ($queue, $proc) = ( 0, 0 );
my $c = shift; my @sysinfo = qw/job worker/;
for my $job ( values %status ) {
my $host = $c->address; $queue += $job->[1];
my $port = $c->args->[0] || 7003; $proc += $job->[2];
}
my $sock = IO::Socket::INET->new(
PeerAddr => $host, return [ $queue, $proc ];
PeerPort => $port, };
);
$sock->syswrite("statusrn");
$sock->sysread( my $raw_status, 8192 );
my %status;
foreach my $line ( split /r?n/, $raw_status ) {
my @st = split /s+/, $line;
next unless @st == 4;
$status{$st[0]} = @st;
}

80. fetcher { my ($queue, $proc) = ( 0, 0 );
my $c = shift; my @sysinfo = qw/job worker/;
for my $job ( values %status ) {
my $host = $c->address; $queue += $job->[1];
my $port = $c->args->[0] || 7003; $proc += $job->[2];
}
my $sock = IO::Socket::INET->new(
PeerAddr => $host, return [ $queue, $proc ];
PeerPort => $port, };
);
$sock->syswrite("statusrn");
$sock->sysread( my $raw_status, 8192 );
my %status;
foreach my $line ( split /r?n/, $raw_status ) {
my @st = split /s+/, $line;
next unless @st == 4;
$status{$st[0]} = @st;
}

81. fetcher { my ($queue, $proc) = ( 0, 0 );
my $c = shift; my @sysinfo = qw/job worker/;
for my $job ( values %status ) {
my $host = $c->address; $queue += $job->[1];
my $port = $c->args->[0] || 7003; $proc += $job->[2];
}
my $sock = IO::Socket::INET->new(
PeerAddr => $host, return [ $queue, $proc ];
PeerPort => $port, };
);
$sock->syswrite("statusrn");
$sock->sysread( my $raw_status, 8192 );
my %status;
foreach my $line ( split /r?n/, $raw_status ) {
my @st = split /s+/, $line;
next unless @st == 4;
$status{$st[0]} = @st;
}

82. # #
title { sysinfo {
my $c = shift; my $c = shift;
my $title = "Gearmand"; #$c->ledge_get('sysinfo') || [];
if ( my $port = $c->args->[0] ) { [‘job’,‘worker’];
$title .= " ($port)"; };
}
return $title;
};
# KVS
$c->ledge_set( 'key', @data, [expires] );
$c->ledge_add( 'key', @data, [expires] );
$c->ledge_get( 'key’ );
$c->ledge_delete( 'key’ );

83. ---
component_config:
resources:
- traffic:eth1
- basic
- gearmand:7003

105. Tips

107. $ vim cloudforecast.yaml
---
config:
gearman_enable: 1
gearman_server:
host: localhost
port: 7003
data_dir: data
host_config_dir: host_config
..

108. #
$ ./cf_fetcher_worker -c cloudforecast.yaml
--max-workers 20
--max-execution-time 60
# RRD
$ ./cf_updater_worker -c cloudforecast.yaml
--max-workers 4

109. #!/bin/sh
exec 2>&1
export CF_DEBUG=1
exec /path/to/cloudforecast/cloudforecast_radar
-r
-c /path/to/cloudforecast/cloudforecast.yaml
-l /path/to/cloudforecast/server_list.yaml