!"#$$%&'()*+&
!"#$%&"'"()*+'%+,%-).++/%0)/1".23"!
                          "#$%!&'!($#)*+!
                           ,-./+,*0012%-'-0.!
                                  4)352($67%
!),,$-&./0&12345&
!!   34,56!7,5008!(,8945$-4!:4;4<08.4%)!=4,.!,)!
     >,*00?!
!!   "8,-*4!7,5008!&0..2@4#!,%5!(4.A4#!0B!C(&!
     -! D$<<1E.4!-0%)#2A$)0#!)0!"8,-*4!7,5008!F2%-4!A2G1A,%G!
        2%!HIIJ!
!"#$$%&6"%7)#38)&9$#":&
!!   K0A=#,-L4#!
     -! (,%,G4F!-<$F)4#!#4F0$#-4F!
        ,%5!M0A!F-*45$<2%G!
!!   =,FL=#,-L4#!
     -! C4#1%054!,G4%)!
     -! (,%,G4!),FLF!
;322)4+&<=0=+">$4?&
!!   N-,<,A2<2)+!
     -! (,O2.$.!&<$F)4#!F2P4!Q!R6III!%054F!
     -! (,O2.$.!-0%-$##4%)!),FLF!Q!RI6III!
     -! &0,#F4!F+%-*#0%2P,E0%!2%!K0A=#,-L4#!
!!   N2%G<4!802%)!0B!B,2<$#4 !!
     -! D,2<$#4!L2<<F!,<<!S$4$45!,%5!#$%%2%G!M0AF!
     -! K0AF!%445!)0!A4!#41F$A.2@45!A+!$F4#F!
!!   94F),#)!2F!;4#+!)#2-L+!5$4!)0!-0.8<4O!F),)4!
!!   7,#5!8,#EE0%!0B!#4F0$#-4F!2%)0!.,8!,%5!#45$-4!
     F<0)F!
;322)4+&<=0=+">$4?&

!!   3,-LF!F$880#)!B0#!,<)4#%,)4!8,#,52G.F!
     -! T)4#,E;4!,88<2-,E0%F!2.8<4.4%)45!$F2%G!(,8945$-4!
        ,#4!UIO!F<0V4#'!!
     -! WO,.8<4X!Y1(4,%F6!C,G49,%L!
!!   3,-L!0B!V2#41-0.8,EA<4!8#0)0-0<F!!
     -! &<24%)!,%5!-<$F)4#!.$F)!A4!0B!F,.4!;4#F20%!
     -! "88<2-,E0%F!,%5!V0#LZ0VF!-,%%0)!.2G#,)4!)0!52[4#4%)
                                                          !
        -<$F)4#F!
6"%7)#38)&'()*+&7)@3=2)0)4+?&A$2&BCDD&
!!   94<2,A2<2)+!
!!   ";,2<,A2<2)+!
!!   N-,<,A2<2)+!1!&<$F)4#F!0B!J6III!.,-*2%4F!
     -! W,-*!.,-*2%4!V2)*!UJ!-0#4F6!R]!9"(6!HR=^!52FLF!
     -! UII6III!-0%-$##4%)!),FLF!
     -! UI6III!-0%-$##4%)!M0AF!
!!   _2#4!&0.8,EA2<2)+!
!!   "G2<2)+!`!W;0<$E0%!Q!"A2<2)+!B0#!-$F)0.4#F!)0!
     -0%)#0<!$8G#,54F!)0!)*4!G#25!F0aV,#4!F),-L'!
6"%7)#38)&'()*+&E&F)?=G4&;)4+2)&
!!   N8<2)!$8!)*4!)V0!.,M0#!B$%-E0%F!0B!K0A=#,-L4#!
     -! &<$F)4#!#4F0$#-4!.,%,G4.4%)!
     -! "88<2-,E0%!<2B41-+-<4!.,%,G4.4%)!
!!   (,8945$-4!A4-0.4F!3?)2H,"4#!<2A#,#+!
128I=+)8+32)&
                                           Node
                                           Node
                                          Manager
                                          Manager


                                    Container   App Mstr
                                                App Mstr


     Client

                         Resource
                         Resource          Node
                                           Node
                         Manager          Manager
                                          Manager
                         Manager
      Client
     Client

                                    App Mstr    Container
                                                Container




      MapReduce Status                     Node
                                           Node
      MapReduce Status                    Manager
                                          Manager
        Job Submission
       Job Submission
         Node Status
        Node Status
      Resource Request
      Resource Request              Container   Container
128I=+)8+32)&
!!   94F0$#-4!(,%,G4#!
     -! ]<0A,<!#4F0$#-4!F-*45$<4#!
     -! 724#,#-*2-,<!S$4$4F!
!!   b054!(,%,G4#!
     -! C4#1.,-*2%4!,G4%)!
     -! (,%,G4F!)*4!<2B41-+-<4!0B!-0%),2%4#!
     -! &0%),2%4#!#4F0$#-4!.0%2)0#2%G!
!!   "88<2-,E0%!(,F)4#!
     -! C4#1,88<2-,E0%!
     -! (,%,G4F!,88<2-,E0%!F-*45$<2%G!,%5!),FL!4O4-$E0%!
     -! W'G'!(,8945$-4!"88<2-,E0%!(,F)4#!
&.0%2$J)0)4+?&J=?HKHJ=?&8322)4+&6"%7)#38)&
  !!   N-,<,A2<2)+!!
       -! "88<2-,E0%!<2B41-+-<4!.,%,G4.4%)!2F!;4#+!
          4O84%F2;4!
       -! C,#EE0%!#4F0$#-4!.,%,G4.4%)!,%5!,88<2-,E0%!
          <2B41-+-<4!.,%,G4.4%)!
       -! "88<2-,E0%!.,%,G4.4%)!2F!52F)#2A$)45!
       -! 7,#5V,#4!)#4%5F!1!&$##4%)<+!#$%!-<$F)4#F!0B!R6III!
          .,-*2%4F!
           c! J6III!HIUH!.,-*2%4F!d!UH6III!HIIe!.,-*2%4F!
           c! f!-0#4F6!UJ]6!R=^d!;gF!fUJh!-0#4F6!RgeJ]6!HR=^d!
&.0%2$J)0)4+?&J=?HKHJ=?&8322)4+&6"%7)#38)&
  !!   ";,2<,A2<2)+!!
       -! "88<2-,E0%!(,F)4#!
           c! i8E0%,<!B,2<0;4#!;2,!,88<2-,E0%1F84-2j-!-*4-L802%)!
           c! (,8945$-4!,88<2-,E0%F!82-L!$8!V*4#4!)*4+!<4a!0[!
       -! 94F0$#-4!(,%,G4#!
           c! b0!F2%G<4!802%)!0B!B,2<$#4!1!B,2<0;4#!;2,!k00Y4484#!
           c! "88<2-,E0%!(,F)4#F!,#4!#4F),#)45!,$)0.,E-,<<+!
&.0%2$J)0)4+?&J=?HKHJ=?&8322)4+&6"%7)#38)&
  !!   _2#4!&0.8,EA2<2)+!!
       -! C#0)0-0<F!,#4!V2#41-0.8,EA<4!
       -! i<5!-<24%)F!-,%!),<L!)0!%4V!F4#;4#F!
       -! 90<<2%G!$8G#,54F!
&.0%2$J)0)4+?&J=?HKHJ=?&8322)4+&6"%7)#38)&
  !!   T%%0;,E0%!,%5!"G2<2)+!
       -! (,8945$-4!%0V!A4-0.4F!,!$F4#1<,%5!<2A#,#+!
       -! ($<E8<4!;4#F20%F!0B!(,8945$-4!-,%!#$%!2%!)*4!
          F,.4!-<$F)4#!l,!<,!"8,-*4!C2Gm!
          c! D,F)4#!548<0+.4%)!-+-<4F!B0#!2.8#0;4.4%)F!
       -! &$F)0.4#F!$8G#,54!(,8945$-4!;4#F20%F!0%!+I)=2&
          F-*45$<4!
       -! nF4#F!-,%!-$F)0.2P4!(,8945$-4!4'G'!7iC!
          V2)*0$)!,[4-E%G!4;4#+0%4?!
&.0%2$J)0)4+?&J=?HKHJ=?&8322)4+&6"%7)#38)&
  !!   nE<2P,E0%!
       -! ]4%4#2-!#4F0$#-4!.054<!!
          c! (4.0#+!
          c! &Cn!
          c! :2FL!AgV!
          c! b4)V0#L!AgV!
       -! 94.0;4!jO45!8,#EE0%!0B!.,8!,%5!#45$-4!F<0)F!
&.0%2$J)0)4+?&J=?HKHJ=?&8322)4+&6"%7)#38)&
  !!   N$880#)!B0#!8#0G#,..2%G!8,#,52G.F!0)*4#!
       )*,%!(,8945$-4!
       -! (CT!
       -! (,F)4#1_0#L4#!
       -! (,-*2%4!34,#%2%G!
       -! T)4#,E;4!8#0-4FF2%G!
       -! W%,A<45!A+!,<<0V2%G!$F4!0B!8,#,52G.1F84-2j-!
          "88<2-,E0%!(,F)4#!
       -! 9$%!,<<!0%!)*4!F,.4!7,5008!-<$F)4#!
L300"2:&
!!   (,8945$-4!'b4O)!),L4F!7,5008!)0!)*4!%4O)!<4;4<!
     -!   N-,<410$)!4;4%!B$#)*4#!
     -!   72G*!,;,2<,A2<2)+!
     -!   &<$F)4#!nE<2P,E0%!!
     -!   N$880#)!B0#!8,#,52G.F!0)*4#!)*,%!(,8945$-4!
M3)?>$4?N&
IO%PQQ#)J),$%)2':"I$$'8$0QR,$G?QI"#$$%Q%$?+?QBCDDQCBQ0"%2)#38)H4)*+G)4Q
                                                                      &

Next Generation MapReduce

  • 1.
    !"#$$%&'()*+& !"#$%&"'"()*+'%+,%-).++/%0)/1".23"! "#$%!&'!($#)*+! ,-./+,*0012%-'-0.! 4)352($67%
  • 2.
    !),,$-&./0&12345& !! 34,56!7,5008!(,8945$-4!:4;4<08.4%)!=4,.!,)! >,*00?! !! "8,-*4!7,5008!&0..2@4#!,%5!(4.A4#!0B!C(&! -! D$<<1E.4!-0%)#2A$)0#!)0!"8,-*4!7,5008!F2%-4!A2G1A,%G! 2%!HIIJ!
  • 3.
    !"#$$%&6"%7)#38)&9$#":& !! K0A=#,-L4#! -! (,%,G4F!-<$F)4#!#4F0$#-4F! ,%5!M0A!F-*45$<2%G! !! =,FL=#,-L4#! -! C4#1%054!,G4%)! -! (,%,G4!),FLF!
  • 4.
    ;322)4+&<=0=+">$4?& !! N-,<,A2<2)+! -! (,O2.$.!&<$F)4#!F2P4!Q!R6III!%054F! -! (,O2.$.!-0%-$##4%)!),FLF!Q!RI6III! -! &0,#F4!F+%-*#0%2P,E0%!2%!K0A=#,-L4#! !! N2%G<4!802%)!0B!B,2<$#4 !! -! D,2<$#4!L2<<F!,<<!S$4$45!,%5!#$%%2%G!M0AF! -! K0AF!%445!)0!A4!#41F$A.2@45!A+!$F4#F! !! 94F),#)!2F!;4#+!)#2-L+!5$4!)0!-0.8<4O!F),)4! !! 7,#5!8,#EE0%!0B!#4F0$#-4F!2%)0!.,8!,%5!#45$-4! F<0)F!
  • 5.
    ;322)4+&<=0=+">$4?& !! 3,-LF!F$880#)!B0#!,<)4#%,)4!8,#,52G.F! -! T)4#,E;4!,88<2-,E0%F!2.8<4.4%)45!$F2%G!(,8945$-4! ,#4!UIO!F<0V4#'!! -! WO,.8<4X!Y1(4,%F6!C,G49,%L! !! 3,-L!0B!V2#41-0.8,EA<4!8#0)0-0<F!! -! &<24%)!,%5!-<$F)4#!.$F)!A4!0B!F,.4!;4#F20%! -! "88<2-,E0%F!,%5!V0#LZ0VF!-,%%0)!.2G#,)4!)0!52[4#4%) ! -<$F)4#F!
  • 6.
    6"%7)#38)&'()*+&7)@3=2)0)4+?&A$2&BCDD& !! 94<2,A2<2)+! !! ";,2<,A2<2)+! !! N-,<,A2<2)+!1!&<$F)4#F!0B!J6III!.,-*2%4F! -! W,-*!.,-*2%4!V2)*!UJ!-0#4F6!R]!9"(6!HR=^!52FLF! -! UII6III!-0%-$##4%)!),FLF! -! UI6III!-0%-$##4%)!M0AF! !! _2#4!&0.8,EA2<2)+! !! "G2<2)+!`!W;0<$E0%!Q!"A2<2)+!B0#!-$F)0.4#F!)0! -0%)#0<!$8G#,54F!)0!)*4!G#25!F0aV,#4!F),-L'!
  • 7.
    6"%7)#38)&'()*+&E&F)?=G4&;)4+2)& !! N8<2)!$8!)*4!)V0!.,M0#!B$%-E0%F!0B!K0A=#,-L4#! -! &<$F)4#!#4F0$#-4!.,%,G4.4%)! -! "88<2-,E0%!<2B41-+-<4!.,%,G4.4%)! !! (,8945$-4!A4-0.4F!3?)2H,"4#!<2A#,#+!
  • 8.
    128I=+)8+32)& Node Node Manager Manager Container App Mstr App Mstr Client Resource Resource Node Node Manager Manager Manager Manager Client Client App Mstr Container Container MapReduce Status Node Node MapReduce Status Manager Manager Job Submission Job Submission Node Status Node Status Resource Request Resource Request Container Container
  • 9.
    128I=+)8+32)& !! 94F0$#-4!(,%,G4#! -! ]<0A,<!#4F0$#-4!F-*45$<4#! -! 724#,#-*2-,<!S$4$4F! !! b054!(,%,G4#! -! C4#1.,-*2%4!,G4%)! -! (,%,G4F!)*4!<2B41-+-<4!0B!-0%),2%4#! -! &0%),2%4#!#4F0$#-4!.0%2)0#2%G! !! "88<2-,E0%!(,F)4#! -! C4#1,88<2-,E0%! -! (,%,G4F!,88<2-,E0%!F-*45$<2%G!,%5!),FL!4O4-$E0%! -! W'G'!(,8945$-4!"88<2-,E0%!(,F)4#!
  • 10.
    &.0%2$J)0)4+?&J=?HKHJ=?&8322)4+&6"%7)#38)& !! N-,<,A2<2)+!! -! "88<2-,E0%!<2B41-+-<4!.,%,G4.4%)!2F!;4#+! 4O84%F2;4! -! C,#EE0%!#4F0$#-4!.,%,G4.4%)!,%5!,88<2-,E0%! <2B41-+-<4!.,%,G4.4%)! -! "88<2-,E0%!.,%,G4.4%)!2F!52F)#2A$)45! -! 7,#5V,#4!)#4%5F!1!&$##4%)<+!#$%!-<$F)4#F!0B!R6III! .,-*2%4F! c! J6III!HIUH!.,-*2%4F!d!UH6III!HIIe!.,-*2%4F! c! f!-0#4F6!UJ]6!R=^d!;gF!fUJh!-0#4F6!RgeJ]6!HR=^d!
  • 11.
    &.0%2$J)0)4+?&J=?HKHJ=?&8322)4+&6"%7)#38)& !! ";,2<,A2<2)+!! -! "88<2-,E0%!(,F)4#! c! i8E0%,<!B,2<0;4#!;2,!,88<2-,E0%1F84-2j-!-*4-L802%)! c! (,8945$-4!,88<2-,E0%F!82-L!$8!V*4#4!)*4+!<4a!0[! -! 94F0$#-4!(,%,G4#! c! b0!F2%G<4!802%)!0B!B,2<$#4!1!B,2<0;4#!;2,!k00Y4484#! c! "88<2-,E0%!(,F)4#F!,#4!#4F),#)45!,$)0.,E-,<<+!
  • 12.
    &.0%2$J)0)4+?&J=?HKHJ=?&8322)4+&6"%7)#38)& !! _2#4!&0.8,EA2<2)+!! -! C#0)0-0<F!,#4!V2#41-0.8,EA<4! -! i<5!-<24%)F!-,%!),<L!)0!%4V!F4#;4#F! -! 90<<2%G!$8G#,54F!
  • 13.
    &.0%2$J)0)4+?&J=?HKHJ=?&8322)4+&6"%7)#38)& !! T%%0;,E0%!,%5!"G2<2)+! -! (,8945$-4!%0V!A4-0.4F!,!$F4#1<,%5!<2A#,#+! -! ($<E8<4!;4#F20%F!0B!(,8945$-4!-,%!#$%!2%!)*4! F,.4!-<$F)4#!l,!<,!"8,-*4!C2Gm! c! D,F)4#!548<0+.4%)!-+-<4F!B0#!2.8#0;4.4%)F! -! &$F)0.4#F!$8G#,54!(,8945$-4!;4#F20%F!0%!+I)=2& F-*45$<4! -! nF4#F!-,%!-$F)0.2P4!(,8945$-4!4'G'!7iC! V2)*0$)!,[4-E%G!4;4#+0%4?!
  • 14.
    &.0%2$J)0)4+?&J=?HKHJ=?&8322)4+&6"%7)#38)& !! nE<2P,E0%! -! ]4%4#2-!#4F0$#-4!.054<!! c! (4.0#+! c! &Cn! c! :2FL!AgV! c! b4)V0#L!AgV! -! 94.0;4!jO45!8,#EE0%!0B!.,8!,%5!#45$-4!F<0)F!
  • 15.
    &.0%2$J)0)4+?&J=?HKHJ=?&8322)4+&6"%7)#38)& !! N$880#)!B0#!8#0G#,..2%G!8,#,52G.F!0)*4#! )*,%!(,8945$-4! -! (CT! -! (,F)4#1_0#L4#! -! (,-*2%4!34,#%2%G! -! T)4#,E;4!8#0-4FF2%G! -! W%,A<45!A+!,<<0V2%G!$F4!0B!8,#,52G.1F84-2j-! "88<2-,E0%!(,F)4#! -! 9$%!,<<!0%!)*4!F,.4!7,5008!-<$F)4#!
  • 16.
    L300"2:& !! (,8945$-4!'b4O)!),L4F!7,5008!)0!)*4!%4O)!<4;4<! -! N-,<410$)!4;4%!B$#)*4#! -! 72G*!,;,2<,A2<2)+! -! &<$F)4#!nE<2P,E0%!! -! N$880#)!B0#!8,#,52G.F!0)*4#!)*,%!(,8945$-4!
  • 17.