Successfully reported this slideshow.
We use your LinkedIn profile and activity data to personalize ads and to show you more relevant ads. You can change your ad preferences anytime.
OGATA Tetsuji (@xtetsuji)
2016/05/19 Gaiax
https://flic.kr/p/pAQruL
• / / OGATA Tetsuji
• @xtetsuji
• Blog: http://post.tetsuji.jp/
• RND
• Qiita:Team
•
•
•
• …
•
• 5 10
•
•
•
•
• 👺 man rm
•
)
• rm
•
•
•
• rm -rf tmp_foo
•
• I/O
•
• I/O
• I/O
$ strace rm moge.pl
execve("/bin/rm", ["rm", "moge.pl"], [/* 52 vars */]) = 0
brk(0) = 0x12c1000
access("/etc/ld.so.nohwca...
•
• find sleep
http://ja.stackoverflow.com/questions/7304/%E5%86%8D%E5%B8%B0%E7%9A%84%E3%81%AB
%E3%82%86%E3%81%A3%E3%81%8F%E3%82%8A%E3%83%...
http://blog.layer8.sh/ja/2011/12/16/%E5%A4%A7%E9%87%8F%E3%81%AE
%E3%83%95%E3%82%A1%E3%82%A4%E3%83%AB%E3%82%92%E3%82%86%E3%...
http://unix.stackexchange.com/questions/106133/why-is-rm-slow
• find xargs
• rsync --delete empty/ garbage/
•
•
•
•
•
• gentle_unlink Perl
•
•
•
•
$ gentle_unlink --help
Usage:
gentle_unlink remove_file_list.txt
ls -1 *.log | gentle_unlink --timeout=600 --progress --in...
• --progress rsync
• --interval
• --timeout
• --flexible
• --dry-run
gentle_unlink
•
•
find $DIR -type f
• --progress
•
• --timeout cron
DEMO?
#!/usr/bin/perl
use strict;
use warnings;
use Carp qw(croak);
use Term::ANSIColor qw(colored);
use Getopt::Long qw(:config...
USR1
$SIG{USR1} = &guess_current_status;
kill -USR1 %1 OK
# for USR1
sub guess_current_status {
my $signal = shift;
printf...
use constant DEBUG => $ENV{DEBUG};
use constant MAX_TIME_HIRES_BIT => 31;
use constant UNLINK_CHUNK_NUM => 20;
use constan...
eval
local $@;
eval {
local $SIG{ALRM} = $timeout > 0 ? sub { $timeout_flag = 1; } : "DEFAULT";
# alarm (10^{-6}) 32 alarm...
sub gentle_unlink {
my @file_chunk;
my $start_time = time;
while (<>) {
chomp;
print "> $_n" if DEBUG;
push @file_chunk, $...
sub unlink_chunk {
my @files = @_;
my @nowhere_files = grep { !-f } @files;
if ( @nowhere_files ) {
if ( $flexible ) {
pro...
sub comma {
my $number = shift;
$number =~ s/(?<=d)(?=(?:ddd)+(?!d))/,/g;
return $number;
}
sub bit {
my $integer = shift;...
•
• cp --reflink=always
•
•
穏やかにファイルを削除する
穏やかにファイルを削除する
穏やかにファイルを削除する
穏やかにファイルを削除する
穏やかにファイルを削除する
Upcoming SlideShare
Loading in …5
×

穏やかにファイルを削除する

3,640 views

Published on

2016/5/19に行われた Gaiaxグループ社内勉強会 で発表したトークのスライドです。

発表時の聴衆コメント → https://slas.la/slide/8

Published in: Technology
  • Be the first to comment

穏やかにファイルを削除する

  1. 1. OGATA Tetsuji (@xtetsuji) 2016/05/19 Gaiax https://flic.kr/p/pAQruL
  2. 2. • / / OGATA Tetsuji • @xtetsuji • Blog: http://post.tetsuji.jp/ • RND
  3. 3. • Qiita:Team • • • • …
  4. 4. • • 5 10 • • •
  5. 5. • • 👺 man rm •
  6. 6. )
  7. 7. • rm •
  8. 8. • • • rm -rf tmp_foo •
  9. 9. • I/O • • I/O • I/O
  10. 10. $ strace rm moge.pl execve("/bin/rm", ["rm", "moge.pl"], [/* 52 vars */]) = 0 brk(0) = 0x12c1000 access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory) mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fd9d1fd3000 access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory) open("/etc/ld.so.cache", O_RDONLY) = 3 fstat(3, {st_mode=S_IFREG|0644, st_size=41893, ...}) = 0 mmap(NULL, 41893, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fd9d1fc8000 close(3) = 0 access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory) open("/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY) = 3 read(3, "177ELF21100000000030>010000360100000"..., 832) = 832 fstat(3, {st_mode=S_IFREG|0755, st_size=1599536, ...}) = 0 mmap(NULL, 3713112, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fd9d1a2b000 mprotect(0x7fd9d1bac000, 2097152, PROT_NONE) = 0 mmap(0x7fd9d1dac000, 20480, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x181000) = 0x7fd9d1dac000 mmap(0x7fd9d1db1000, 18520, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fd9d1db1000 close(3) = 0 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fd9d1fc7000 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fd9d1fc6000 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fd9d1fc5000 arch_prctl(ARCH_SET_FS, 0x7fd9d1fc6700) = 0 mprotect(0x7fd9d1dac000, 16384, PROT_READ) = 0 mprotect(0x60d000, 4096, PROT_READ) = 0 mprotect(0x7fd9d1fd5000, 4096, PROT_READ) = 0 munmap(0x7fd9d1fc8000, 41893) = 0 brk(0) = 0x12c1000 brk(0x12e2000) = 0x12e2000 open("/usr/lib/locale/locale-archive", O_RDONLY) = 3 fstat(3, {st_mode=S_IFREG|0644, st_size=2508736, ...}) = 0 mmap(NULL, 2508736, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fd9d17c6000 close(3) = 0 ioctl(0, SNDCTL_TMR_TIMEBASE or TCGETS, {B9600 opost isig icanon echo ...}) = 0 newfstatat(AT_FDCWD, "moge.pl", {st_mode=S_IFREG|0755, st_size=0, ...}, AT_SYMLINK_NOFOLLOW) = 0 geteuid() = 1000 newfstatat(AT_FDCWD, "moge.pl", {st_mode=S_IFREG|0755, st_size=0, ...}, AT_SYMLINK_NOFOLLOW) = 0 faccessat(AT_FDCWD, "moge.pl", W_OK) = 0 unlinkat(AT_FDCWD, "moge.pl", 0) = 0 lseek(0, 0, SEEK_CUR) = -1 ESPIPE (Illegal seek) close(0) = 0 close(1) = 0 close(2) = 0 exit_group(0) = ?
  11. 11. • • find sleep
  12. 12. http://ja.stackoverflow.com/questions/7304/%E5%86%8D%E5%B8%B0%E7%9A%84%E3%81%AB %E3%82%86%E3%81%A3%E3%81%8F%E3%82%8A%E3%83%95%E3%82%A1%E3%82%A4%E3%83%AB %E3%81%A0%E3%81%91%E3%82%92%E5%89%8A%E9%99%A4%E3%81%99%E3%82%8B %E3%82%B3%E3%83%9E%E3%83%B3%E3%83%89
  13. 13. http://blog.layer8.sh/ja/2011/12/16/%E5%A4%A7%E9%87%8F%E3%81%AE %E3%83%95%E3%82%A1%E3%82%A4%E3%83%AB%E3%82%92%E3%82%86%E3%81%A3%E3%81%8F %E3%82%8A%E7%9B%AE%E3%81%AB%E5%89%8A%E9%99%A4%E3%81%99/
  14. 14. http://unix.stackexchange.com/questions/106133/why-is-rm-slow
  15. 15. • find xargs • rsync --delete empty/ garbage/ •
  16. 16. • • • •
  17. 17. • gentle_unlink Perl • • • •
  18. 18. $ gentle_unlink --help Usage: gentle_unlink remove_file_list.txt ls -1 *.log | gentle_unlink --timeout=600 --progress --interval=2 gentle_unlink [--timeout=SECONDS] [--progress] [--interval=SECONDS] [--flexible] [--dry-run]
  19. 19. • --progress rsync • --interval • --timeout • --flexible • --dry-run
  20. 20. gentle_unlink • • find $DIR -type f • --progress • • --timeout cron
  21. 21. DEMO?
  22. 22. #!/usr/bin/perl use strict; use warnings; use Carp qw(croak); use Term::ANSIColor qw(colored); use Getopt::Long qw(:config posix_default no_ignore_case bundling auto_help); use Time::HiRes qw(time usleep ualarm);
  23. 23. USR1 $SIG{USR1} = &guess_current_status; kill -USR1 %1 OK # for USR1 sub guess_current_status { my $signal = shift; printf colored("dry-run mode. all delete operations are fake.n", "blue") if $dry_run; printf "process id: %dn", $$; printf "delete file count: %sn", comma($delete_count); printf "delete size total: %s bytesn", comma($delete_size_total); printf "process run times: %s secondn", comma(int(time - $process_start_time)); printf "interval=%d timeout=%sn", $interval, $timeout; }
  24. 24. use constant DEBUG => $ENV{DEBUG}; use constant MAX_TIME_HIRES_BIT => 31; use constant UNLINK_CHUNK_NUM => 20; use constant COLOR_WARNING => "yellow"; use constant COLOR_ERROR => "red"; use constant COLOR_INFO => "green"; $SIG{USR1} = &guess_current_status; GetOptions( my %opt, "timeout|t=f", "progress|p", "interval|i=f", "flexible|f", "dry-run|n" ); my $timeout = $opt{timeout} || 0; my $progress = $opt{progress}; my $interval = $opt{interval} || 0; my $dry_run = $opt{"dry-run"}; my $flexible = $opt{flexible}; my $delete_count = 0; my $delete_size_total = 0; my $timeout_flag; my $preserve_interval_flag; my $process_start_time = time;
  25. 25. eval local $@; eval { local $SIG{ALRM} = $timeout > 0 ? sub { $timeout_flag = 1; } : "DEFAULT"; # alarm (10^{-6}) 32 alarm my $alarm_cancel_cb; # alram 0 ualarm 0 my $timeout_microsecond = $timeout * 10**6; if ( bit($timeout_microsecond) < MAX_TIME_HIRES_BIT ) { ualarm($timeout_microsecond); $alarm_cancel_cb = sub { ualarm 0; }; } else { alarm(int($timeout)); $alarm_cancel_cb = sub { alarm 0; }; } gentle_unlink(); $alarm_cancel_cb->(); }; if ( $@ ) { print colored("e=$@n", COLOR_ERROR); }
  26. 26. sub gentle_unlink { my @file_chunk; my $start_time = time; while (<>) { chomp; print "> $_n" if DEBUG; push @file_chunk, $_; if ( @file_chunk > UNLINK_CHUNK_NUM ) { unlink_chunk(@file_chunk); @file_chunk = (); $preserve_interval_flag = 1; } } continue { if ( $timeout_flag ) { progress_printf(colored("timeoutn", COLOR_WARNING)); last; } if ( $interval && $preserve_interval_flag ) { progress_printf(colored("interval %f seconds.n", COLOR_INFO), $interval ); usleep $interval * 10**6; $preserve_interval_flag = 0; } } if ( @file_chunk ) { unlink_chunk(@file_chunk); } my $end_time = time; progress_printf("%f seconds, %s bytes, total %d files deleted.n", $end_time - $start_time, comma($delete_size_total), $delete_count); progress_printf(colored("dry-run mode. all delete operations are fake.n", "blue")) if $dry_run; }
  27. 27. sub unlink_chunk { my @files = @_; my @nowhere_files = grep { !-f } @files; if ( @nowhere_files ) { if ( $flexible ) { progress_printf(colored( join( "", map { qq(file "$_" is not foundn) } @nowhere_files ), COLOR_WARNING )); # progress_printf colored } else { croak colored("unlink_chunk gives lost filename.", COLOR_ERROR); } } progress_printf(">>> dry-run moden") if $dry_run; progress_printf("%4d: deleting... %s (%s bytes)n", ++$delete_count, $_, comma(-s $_)) for @files; $delete_size_total += -s $_ for @files; return if $dry_run; local $!; my $chunk_num = @files; my $delete_num = unlink @files; if ( $chunk_num != $delete_num ) { print colored("maybe delete failed.n" . "w=$!", COLOR_ERROR); croak colored("delete failure error", COLOR_ERROR) if !$flexible; } }
  28. 28. sub comma { my $number = shift; $number =~ s/(?<=d)(?=(?:ddd)+(?!d))/,/g; return $number; } sub bit { my $integer = shift; return 0 if $integer == 0; return( log($integer) / log(2) ); }
  29. 29. • • cp --reflink=always • •

×