Successfully reported this slideshow.
Your SlideShare is downloading. ×

穏やかにファイルを削除する

Ad
Ad
Ad
Ad
Ad
Ad
Ad
Ad
Ad
Ad
Ad
Upcoming SlideShare
PHP Tutorial (funtion)
PHP Tutorial (funtion)
Loading in …3
×

Check these out next

1 of 34 Ad

穏やかにファイルを削除する

Download to read offline

2016/5/19に行われた Gaiaxグループ社内勉強会 で発表したトークのスライドです。

発表時の聴衆コメント → https://slas.la/slide/8

2016/5/19に行われた Gaiaxグループ社内勉強会 で発表したトークのスライドです。

発表時の聴衆コメント → https://slas.la/slide/8

Advertisement
Advertisement

More Related Content

Slideshows for you (20)

Similar to 穏やかにファイルを削除する (20)

Advertisement

More from 鉄次 尾形 (20)

Recently uploaded (20)

Advertisement

穏やかにファイルを削除する

  1. 1. OGATA Tetsuji (@xtetsuji) 2016/05/19 Gaiax https://flic.kr/p/pAQruL
  2. 2. • / / OGATA Tetsuji • @xtetsuji • Blog: http://post.tetsuji.jp/ • RND
  3. 3. • Qiita:Team • • • • …
  4. 4. • • 5 10 • • •
  5. 5. • • 👺 man rm •
  6. 6. )
  7. 7. • rm •
  8. 8. • • • rm -rf tmp_foo •
  9. 9. • I/O • • I/O • I/O
  10. 10. $ strace rm moge.pl execve("/bin/rm", ["rm", "moge.pl"], [/* 52 vars */]) = 0 brk(0) = 0x12c1000 access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory) mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fd9d1fd3000 access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory) open("/etc/ld.so.cache", O_RDONLY) = 3 fstat(3, {st_mode=S_IFREG|0644, st_size=41893, ...}) = 0 mmap(NULL, 41893, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fd9d1fc8000 close(3) = 0 access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory) open("/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY) = 3 read(3, "177ELF21100000000030>010000360100000"..., 832) = 832 fstat(3, {st_mode=S_IFREG|0755, st_size=1599536, ...}) = 0 mmap(NULL, 3713112, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fd9d1a2b000 mprotect(0x7fd9d1bac000, 2097152, PROT_NONE) = 0 mmap(0x7fd9d1dac000, 20480, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x181000) = 0x7fd9d1dac000 mmap(0x7fd9d1db1000, 18520, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fd9d1db1000 close(3) = 0 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fd9d1fc7000 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fd9d1fc6000 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fd9d1fc5000 arch_prctl(ARCH_SET_FS, 0x7fd9d1fc6700) = 0 mprotect(0x7fd9d1dac000, 16384, PROT_READ) = 0 mprotect(0x60d000, 4096, PROT_READ) = 0 mprotect(0x7fd9d1fd5000, 4096, PROT_READ) = 0 munmap(0x7fd9d1fc8000, 41893) = 0 brk(0) = 0x12c1000 brk(0x12e2000) = 0x12e2000 open("/usr/lib/locale/locale-archive", O_RDONLY) = 3 fstat(3, {st_mode=S_IFREG|0644, st_size=2508736, ...}) = 0 mmap(NULL, 2508736, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fd9d17c6000 close(3) = 0 ioctl(0, SNDCTL_TMR_TIMEBASE or TCGETS, {B9600 opost isig icanon echo ...}) = 0 newfstatat(AT_FDCWD, "moge.pl", {st_mode=S_IFREG|0755, st_size=0, ...}, AT_SYMLINK_NOFOLLOW) = 0 geteuid() = 1000 newfstatat(AT_FDCWD, "moge.pl", {st_mode=S_IFREG|0755, st_size=0, ...}, AT_SYMLINK_NOFOLLOW) = 0 faccessat(AT_FDCWD, "moge.pl", W_OK) = 0 unlinkat(AT_FDCWD, "moge.pl", 0) = 0 lseek(0, 0, SEEK_CUR) = -1 ESPIPE (Illegal seek) close(0) = 0 close(1) = 0 close(2) = 0 exit_group(0) = ?
  11. 11. • • find sleep
  12. 12. http://ja.stackoverflow.com/questions/7304/%E5%86%8D%E5%B8%B0%E7%9A%84%E3%81%AB %E3%82%86%E3%81%A3%E3%81%8F%E3%82%8A%E3%83%95%E3%82%A1%E3%82%A4%E3%83%AB %E3%81%A0%E3%81%91%E3%82%92%E5%89%8A%E9%99%A4%E3%81%99%E3%82%8B %E3%82%B3%E3%83%9E%E3%83%B3%E3%83%89
  13. 13. http://blog.layer8.sh/ja/2011/12/16/%E5%A4%A7%E9%87%8F%E3%81%AE %E3%83%95%E3%82%A1%E3%82%A4%E3%83%AB%E3%82%92%E3%82%86%E3%81%A3%E3%81%8F %E3%82%8A%E7%9B%AE%E3%81%AB%E5%89%8A%E9%99%A4%E3%81%99/
  14. 14. http://unix.stackexchange.com/questions/106133/why-is-rm-slow
  15. 15. • find xargs • rsync --delete empty/ garbage/ •
  16. 16. • • • •
  17. 17. • gentle_unlink Perl • • • •
  18. 18. $ gentle_unlink --help Usage: gentle_unlink remove_file_list.txt ls -1 *.log | gentle_unlink --timeout=600 --progress --interval=2 gentle_unlink [--timeout=SECONDS] [--progress] [--interval=SECONDS] [--flexible] [--dry-run]
  19. 19. • --progress rsync • --interval • --timeout • --flexible • --dry-run
  20. 20. gentle_unlink • • find $DIR -type f • --progress • • --timeout cron
  21. 21. DEMO?
  22. 22. #!/usr/bin/perl use strict; use warnings; use Carp qw(croak); use Term::ANSIColor qw(colored); use Getopt::Long qw(:config posix_default no_ignore_case bundling auto_help); use Time::HiRes qw(time usleep ualarm);
  23. 23. USR1 $SIG{USR1} = &guess_current_status; kill -USR1 %1 OK # for USR1 sub guess_current_status { my $signal = shift; printf colored("dry-run mode. all delete operations are fake.n", "blue") if $dry_run; printf "process id: %dn", $$; printf "delete file count: %sn", comma($delete_count); printf "delete size total: %s bytesn", comma($delete_size_total); printf "process run times: %s secondn", comma(int(time - $process_start_time)); printf "interval=%d timeout=%sn", $interval, $timeout; }
  24. 24. use constant DEBUG => $ENV{DEBUG}; use constant MAX_TIME_HIRES_BIT => 31; use constant UNLINK_CHUNK_NUM => 20; use constant COLOR_WARNING => "yellow"; use constant COLOR_ERROR => "red"; use constant COLOR_INFO => "green"; $SIG{USR1} = &guess_current_status; GetOptions( my %opt, "timeout|t=f", "progress|p", "interval|i=f", "flexible|f", "dry-run|n" ); my $timeout = $opt{timeout} || 0; my $progress = $opt{progress}; my $interval = $opt{interval} || 0; my $dry_run = $opt{"dry-run"}; my $flexible = $opt{flexible}; my $delete_count = 0; my $delete_size_total = 0; my $timeout_flag; my $preserve_interval_flag; my $process_start_time = time;
  25. 25. eval local $@; eval { local $SIG{ALRM} = $timeout > 0 ? sub { $timeout_flag = 1; } : "DEFAULT"; # alarm (10^{-6}) 32 alarm my $alarm_cancel_cb; # alram 0 ualarm 0 my $timeout_microsecond = $timeout * 10**6; if ( bit($timeout_microsecond) < MAX_TIME_HIRES_BIT ) { ualarm($timeout_microsecond); $alarm_cancel_cb = sub { ualarm 0; }; } else { alarm(int($timeout)); $alarm_cancel_cb = sub { alarm 0; }; } gentle_unlink(); $alarm_cancel_cb->(); }; if ( $@ ) { print colored("e=$@n", COLOR_ERROR); }
  26. 26. sub gentle_unlink { my @file_chunk; my $start_time = time; while (<>) { chomp; print "> $_n" if DEBUG; push @file_chunk, $_; if ( @file_chunk > UNLINK_CHUNK_NUM ) { unlink_chunk(@file_chunk); @file_chunk = (); $preserve_interval_flag = 1; } } continue { if ( $timeout_flag ) { progress_printf(colored("timeoutn", COLOR_WARNING)); last; } if ( $interval && $preserve_interval_flag ) { progress_printf(colored("interval %f seconds.n", COLOR_INFO), $interval ); usleep $interval * 10**6; $preserve_interval_flag = 0; } } if ( @file_chunk ) { unlink_chunk(@file_chunk); } my $end_time = time; progress_printf("%f seconds, %s bytes, total %d files deleted.n", $end_time - $start_time, comma($delete_size_total), $delete_count); progress_printf(colored("dry-run mode. all delete operations are fake.n", "blue")) if $dry_run; }
  27. 27. sub unlink_chunk { my @files = @_; my @nowhere_files = grep { !-f } @files; if ( @nowhere_files ) { if ( $flexible ) { progress_printf(colored( join( "", map { qq(file "$_" is not foundn) } @nowhere_files ), COLOR_WARNING )); # progress_printf colored } else { croak colored("unlink_chunk gives lost filename.", COLOR_ERROR); } } progress_printf(">>> dry-run moden") if $dry_run; progress_printf("%4d: deleting... %s (%s bytes)n", ++$delete_count, $_, comma(-s $_)) for @files; $delete_size_total += -s $_ for @files; return if $dry_run; local $!; my $chunk_num = @files; my $delete_num = unlink @files; if ( $chunk_num != $delete_num ) { print colored("maybe delete failed.n" . "w=$!", COLOR_ERROR); croak colored("delete failure error", COLOR_ERROR) if !$flexible; } }
  28. 28. sub comma { my $number = shift; $number =~ s/(?<=d)(?=(?:ddd)+(?!d))/,/g; return $number; } sub bit { my $integer = shift; return 0 if $integer == 0; return( log($integer) / log(2) ); }
  29. 29. • • cp --reflink=always • •

×