2016/5/19に行われた Gaiaxグループ社内勉強会 で発表したトークのスライドです。 発表時の聴衆コメント → https://slas.la/slide/8

1. OGATA Tetsuji (@xtetsuji)
2016/05/19 Gaiax
https://flic.kr/p/pAQruL

2. • / / OGATA Tetsuji
• @xtetsuji
• Blog: http://post.tetsuji.jp/
• RND

3. • Qiita:Team
•
•
•
• …

4. •
• 5 10
•
•
•

6. •
• 👺 man rm
•

7. )

8. • rm
•

9. •
•
• rm -rf tmp_foo
•

10. • I/O
•
• I/O
• I/O

11. $ strace rm moge.pl
execve("/bin/rm", ["rm", "moge.pl"], [/* 52 vars */]) = 0
brk(0) = 0x12c1000
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fd9d1fd3000
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=41893, ...}) = 0
mmap(NULL, 41893, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fd9d1fc8000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY) = 3
read(3, "177ELF21100000000030>010000360100000"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1599536, ...}) = 0
mmap(NULL, 3713112, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fd9d1a2b000
mprotect(0x7fd9d1bac000, 2097152, PROT_NONE) = 0
mmap(0x7fd9d1dac000, 20480, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x181000) = 0x7fd9d1dac000
mmap(0x7fd9d1db1000, 18520, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fd9d1db1000
close(3) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fd9d1fc7000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fd9d1fc6000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fd9d1fc5000
arch_prctl(ARCH_SET_FS, 0x7fd9d1fc6700) = 0
mprotect(0x7fd9d1dac000, 16384, PROT_READ) = 0
mprotect(0x60d000, 4096, PROT_READ) = 0
mprotect(0x7fd9d1fd5000, 4096, PROT_READ) = 0
munmap(0x7fd9d1fc8000, 41893) = 0
brk(0) = 0x12c1000
brk(0x12e2000) = 0x12e2000
open("/usr/lib/locale/locale-archive", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=2508736, ...}) = 0
mmap(NULL, 2508736, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fd9d17c6000
close(3) = 0
ioctl(0, SNDCTL_TMR_TIMEBASE or TCGETS, {B9600 opost isig icanon echo ...}) = 0
newfstatat(AT_FDCWD, "moge.pl", {st_mode=S_IFREG|0755, st_size=0, ...}, AT_SYMLINK_NOFOLLOW) = 0
geteuid() = 1000
newfstatat(AT_FDCWD, "moge.pl", {st_mode=S_IFREG|0755, st_size=0, ...}, AT_SYMLINK_NOFOLLOW) = 0
faccessat(AT_FDCWD, "moge.pl", W_OK) = 0
unlinkat(AT_FDCWD, "moge.pl", 0) = 0
lseek(0, 0, SEEK_CUR) = -1 ESPIPE (Illegal seek)
close(0) = 0
close(1) = 0
close(2) = 0
exit_group(0) = ?

12. •
• find sleep

13. http://ja.stackoverflow.com/questions/7304/%E5%86%8D%E5%B8%B0%E7%9A%84%E3%81%AB
%E3%82%86%E3%81%A3%E3%81%8F%E3%82%8A%E3%83%95%E3%82%A1%E3%82%A4%E3%83%AB
%E3%81%A0%E3%81%91%E3%82%92%E5%89%8A%E9%99%A4%E3%81%99%E3%82%8B
%E3%82%B3%E3%83%9E%E3%83%B3%E3%83%89

14. http://blog.layer8.sh/ja/2011/12/16/%E5%A4%A7%E9%87%8F%E3%81%AE
%E3%83%95%E3%82%A1%E3%82%A4%E3%83%AB%E3%82%92%E3%82%86%E3%81%A3%E3%81%8F
%E3%82%8A%E7%9B%AE%E3%81%AB%E5%89%8A%E9%99%A4%E3%81%99/

15. http://unix.stackexchange.com/questions/106133/why-is-rm-slow

16. • find xargs
• rsync --delete empty/ garbage/
•

17. •
•
•
•

18. • gentle_unlink Perl
•
•
•
•

19. $ gentle_unlink --help
Usage:
gentle_unlink remove_file_list.txt
ls -1 *.log | gentle_unlink --timeout=600 --progress --interval=2
gentle_unlink [--timeout=SECONDS] [--progress] [--interval=SECONDS]
[--flexible] [--dry-run]

20. • --progress rsync
• --interval
• --timeout
• --flexible
• --dry-run

21. gentle_unlink
•
•
find $DIR -type f
• --progress
•
• --timeout cron

22. DEMO?

24. #!/usr/bin/perl
use strict;
use warnings;
use Carp qw(croak);
use Term::ANSIColor qw(colored);
use Getopt::Long qw(:config posix_default no_ignore_case bundling auto_help);
use Time::HiRes qw(time usleep ualarm);

25. USR1
$SIG{USR1} = &guess_current_status;
kill -USR1 %1 OK
# for USR1
sub guess_current_status {
my $signal = shift;
printf colored("dry-run mode. all delete operations are fake.n", "blue") if $dry_run;
printf "process id: %dn", $$;
printf "delete file count: %sn", comma($delete_count);
printf "delete size total: %s bytesn", comma($delete_size_total);
printf "process run times: %s secondn", comma(int(time - $process_start_time));
printf "interval=%d timeout=%sn", $interval, $timeout;
}

26. use constant DEBUG => $ENV{DEBUG};
use constant MAX_TIME_HIRES_BIT => 31;
use constant UNLINK_CHUNK_NUM => 20;
use constant COLOR_WARNING => "yellow";
use constant COLOR_ERROR => "red";
use constant COLOR_INFO => "green";
$SIG{USR1} = &guess_current_status;
GetOptions(
my %opt,
"timeout|t=f", "progress|p", "interval|i=f", "flexible|f", "dry-run|n"
);
my $timeout = $opt{timeout} || 0;
my $progress = $opt{progress};
my $interval = $opt{interval} || 0;
my $dry_run = $opt{"dry-run"};
my $flexible = $opt{flexible};
my $delete_count = 0;
my $delete_size_total = 0;
my $timeout_flag;
my $preserve_interval_flag;
my $process_start_time = time;

27. eval
local $@;
eval {
local $SIG{ALRM} = $timeout > 0 ? sub { $timeout_flag = 1; } : "DEFAULT";
# alarm (10^{-6}) 32 alarm
my $alarm_cancel_cb; # alram 0 ualarm 0
my $timeout_microsecond = $timeout * 10**6;
if ( bit($timeout_microsecond) < MAX_TIME_HIRES_BIT ) {
ualarm($timeout_microsecond);
$alarm_cancel_cb = sub { ualarm 0; };
} else {
alarm(int($timeout));
$alarm_cancel_cb = sub { alarm 0; };
}
gentle_unlink();
$alarm_cancel_cb->();
};
if ( $@ ) {
print colored("e=$@n", COLOR_ERROR);
}

28. sub gentle_unlink {
my @file_chunk;
my $start_time = time;
while (<>) {
chomp;
print "> $_n" if DEBUG;
push @file_chunk, $_;
if ( @file_chunk > UNLINK_CHUNK_NUM ) {
unlink_chunk(@file_chunk);
@file_chunk = ();
$preserve_interval_flag = 1;
}
} continue {
if ( $timeout_flag ) {
progress_printf(colored("timeoutn", COLOR_WARNING));
last;
}
if ( $interval && $preserve_interval_flag ) {
progress_printf(colored("interval %f seconds.n", COLOR_INFO), $interval );
usleep $interval * 10**6;
$preserve_interval_flag = 0;
}
}
if ( @file_chunk ) {
unlink_chunk(@file_chunk);
}
my $end_time = time;
progress_printf("%f seconds, %s bytes, total %d files deleted.n",
$end_time - $start_time, comma($delete_size_total), $delete_count);
progress_printf(colored("dry-run mode. all delete operations are fake.n", "blue")) if
$dry_run;
}

29. sub unlink_chunk {
my @files = @_;
my @nowhere_files = grep { !-f } @files;
if ( @nowhere_files ) {
if ( $flexible ) {
progress_printf(colored(
join( "", map { qq(file "$_" is not foundn) } @nowhere_files ),
COLOR_WARNING
)); # progress_printf colored
} else {
croak colored("unlink_chunk gives lost filename.", COLOR_ERROR);
}
}
progress_printf(">>> dry-run moden") if $dry_run;
progress_printf("%4d: deleting... %s (%s bytes)n",
++$delete_count, $_, comma(-s $_)) for @files;
$delete_size_total += -s $_ for @files;
return if $dry_run;
local $!;
my $chunk_num = @files;
my $delete_num = unlink @files;
if ( $chunk_num != $delete_num ) {
print colored("maybe delete failed.n" . "w=$!", COLOR_ERROR);
croak colored("delete failure error", COLOR_ERROR) if !$flexible;
}
}

30. sub comma {
my $number = shift;
$number =~ s/(?<=d)(?=(?:ddd)+(?!d))/,/g;
return $number;
}
sub bit {
my $integer = shift;
return 0 if $integer == 0;
return( log($integer) / log(2) );
}

33. •
• cp --reflink=always
•
•