RHadoop, R meets Hadoop

RHadoop, Hadoop for R

Scholarly Activity 05-09 change 50%37.5% 25%12.5% 0%-12.5% -25%-37.5% R SAS SPSS S-Plus Stata

Scholarly Activity 05-09 change 50% 37.5% 25% 12.5% Packages 0%10000 -12.5% -25% 1000 -37.5% R SAS SPSS S-Plus Stata 100 10 1 2002 2004 2006 2008 2010

Scholarly Activity 05-09 change 50% 37.5% 25% 12.5% Packages 0%10000 -12.5% -25% 1000 -37.5% R SAS SPSS S-Plus Stata 100 10 http://r4stats.com/popularity 1 2002 2004 2006 2008 2010

David Champagne, CTO

f s h dr

rh d f srhb ase

rh d f srhb ase rm r

sapply(data, function)mapreduce(data, map = function)

library(rmr)mapreduce(…)

Rmr Java, C++

Rmr Cascading, Java, C++ Crunch

Rmr, Rhipe, Dumbo,Rmr Pydoop, Hadoopy Cascading, Java, C++ Crunch

Expose MR Hide MRRmr, Rhipe, Dumbo,Rmr Pydoop, Hadoopy Cascading, Java, C++ Crunch

Expose MR Hide MR Hive, PigRmr, Rhipe, Dumbo,Rmr Pydoop, Hadoopy Cascading, Java, C++ Crunch

Expose MR Hide MR Hive, PigRmr, Rhipe, Dumbo,Rmr Cascalog, Pydoop, Hadoopy Scalding, Scrunch Cascading, Java, C++ Crunch

mapreduce(input, output, map, reduce)

map = function(k, v) if (hash(k) %% 10 == 0)keyval(k, v) reduce = function(k, vv) keyval(k, length(vv))

condition = function(x) x > 10

condition = function(x) x > 10out = mapreduce(

condition = function(x) x > 10out = mapreduce( input = input,

condition = function(x) x > 10out = mapreduce( input = input, map = function(k,v)

condition = function(x) x > 10out = mapreduce( input = input, map = function(k,v) if (condition(v)) keyval(k,v))

x = from.dfs(hdfs.object)hdfs.object = to.dfs(x)

INSERT OVERWRITE TABLE pv_gender_sumSELECT pv_users.gender, count (DISTINCT pv_users.userid)FROM pv_usersGROUP BY pv_users.gender;

INSERT OVERWRITE TABLE pv_gender_sumSELECT pv_users.gender, count (DISTINCT pv_users.userid)FROM pv_usersGROUP BY pv_users.gender;mapreduce(input = mapreduce(input = "pv_users", map = function(k, v) keyval(v[userid], v[gender]), reduce = function(uid, genders) lapply(unique(genders), function(g) keyval(NULL, g)), output = "pv_gender_sum", map = function(x, gender) keyval(gender, 1) reduce = function(gender,counts) keyval(k,sum(unlist(counts)))

kmeans = function(points, ncenters, iterations = 10, distfun = function(a,b) norm(as.matrix(a-b), type = F)) { newCenters = kmeans.iter(points, distfun, ncenters = ncenters) for(i in 1:iterations) { newCenters = kmeans.iter(points, distfun, centers = newCenters)} newCenters}

kmeans = function(points, ncenters, iterations = 10, distfun = function(a,b) norm(as.matrix(a-b), type = F)) { newCenters = kmeans.iter(points, distfun, ncenters = ncenters) for(i in 1:iterations) { newCenters = kmeans.iter(points, distfun, centers = newCenters)} newCenters}kmeans.iter = function(points, distfun, ncenters = dim(centers)[1], centers = NULL) { from.dfs( mapreduce( input = points, map = if (is.null(centers)) { function(k,v) keyval(sample(1:ncenters,1),v)} else { function(k,v) { distances = apply(centers, 1, function(c) distfun(c,v)) keyval(centers[which.min(distances),], v)}}, reduce = function(k,vv) keyval(NULL, apply(do.call(rbind, vv), 2, mean))), to.data.frame = T)}

#!/usr/bin/pythonimport sysfrom math import fabsfrom org.apache.pig.scripting import Pigfilename = "student.txt"k = 4tolerance = 0.01MAX_SCORE = 4MIN_SCORE = 0MAX_ITERATION = 100# initial centroid, equally divide the spaceinitial_centroids = ""last_centroids = [None] * kfor i in range(k): last_centroids[i] = MIN_SCORE + float(i)/k*(MAX_SCORE-MIN_SCORE) initial_centroids = initial_centroids + str(last_centroids[i]) if i!=k-1: initial_centroids = initial_centroids + ":"P = Pig.compile("""register udf.jar DEFINE find_centroid FindCentroid($centroids); raw = load student.txt as (name:chararray, age:int, gpa:double); centroided = foreach raw generate gpa, find_centroid(gpa) as centroid; grouped = group centroided by centroid; result = foreach grouped generate group, AVG(centroided.gpa); store result into output; """)converged = Falseiter_num = 0while iter_num<MAX_ITERATION: Q = P.bind({centroids:initial_centroids}) results = Q.runSingle()

if results.isSuccessful() == "FAILED": raise "Pig job failed" iter = results.result("result").iterator() centroids = [None] * k distance_move = 0 # get new centroid of this iteration, caculate the moving distance with last iteration for i in range(k): tuple = iter.next() centroids[i] = float(str(tuple.get(1))) distance_move = distance_move + fabs(last_centroids[i]-centroids[i]) distance_move = distance_move / k; Pig.fs("rmr output") print("iteration " + str(iter_num)) print("average distance moved: " + str(distance_move)) if distance_move<tolerance: sys.stdout.write("k-means converged at centroids: [") sys.stdout.write(",".join(str(v) for v in centroids)) sys.stdout.write("]n") converged = True break last_centroids = centroids[:] initial_centroids = "" for i in range(k): initial_centroids = initial_centroids + str(last_centroids[i]) if i!=k-1: initial_centroids = initial_centroids + ":" iter_num += 1if not converged: print("not converge after " + str(iter_num) + " iterations") sys.stdout.write("last centroids: [") sys.stdout.write(",".join(str(v) for v in last_centroids)) sys.stdout.write("]n")

import java.io.IOException;import org.apache.pig.EvalFunc;import org.apache.pig.data.Tuple;public class FindCentroid extends EvalFunc<Double> { double[] centroids; public FindCentroid(String initialCentroid) { String[] centroidStrings = initialCentroid.split(":"); centroids = new double[centroidStrings.length]; for (int i=0;i<centroidStrings.length;i++) centroids[i] = Double.parseDouble(centroidStrings[i]); } @Override public Double exec(Tuple input) throws IOException { double min_distance = Double.MAX_VALUE; double closest_centroid = 0; for (double centroid : centroids) { double distance = Math.abs(centroid - (Double)input.get(0)); if (distance < min_distance) { min_distance = distance; closest_centroid = centroid; } } return closest_centroid; }}

mapreduce(mapreduce(…

mapreduce(mapreduce(…mapreduce(input = c(input1, input2), …)

mapreduce(mapreduce(…mapreduce(input = c(input1, input2), …)equijoin = function( left.input, right.input, input, output, outer, map.left, map.right, reduce, reduce.all)

out1 = mapreduce(…)mapreduce(input = out1, <xyz>)mapreduce(input = out1, <abc>)

out1 = mapreduce(…)mapreduce(input = out1, <xyz>)mapreduce(input = out1, <abc>)abstract.job = function(input, output, …) { … result = mapreduce(input = input, output = output) … result}

input.format, output.format, format

input.format, output.format, formatcombine

input.format, output.format, formatcombinereduce.on.data.frame

input.format, output.format, formatcombinereduce.on.data.framelocal, hadoop backends

input.format, output.format, formatcombinereduce.on.data.framelocal, hadoop backendsbackend.parameters

input.format, output.format, formatcombinereduce.on.data.framelocal, hadoop backendsbackend.parametersprofiling

input.format, output.format, formatcombinereduce.on.data.framelocal, hadoop backendsbackend.parametersprofilingverbose

RHADOOP USERONE FAT CLUSTER AVE.HYDROPOWER CITY, OR 0x0000 RHADOOP@ REVOLUTIONANALYTICS.COM

http://blog.revolutionanalytics.com	31264
http://www.r-bloggers.com	2267
http://smartdatacollective.com	660
http://www.scoop.it	43
http://feeds.feedburner.com	20
http://translate.googleusercontent.com	19
http://www.newsblur.com	15
http://atomicules.co.uk	12
http://webcache.googleusercontent.com	9
http://vizdat.collected.info	9
http://core.traackr.com	5
http://03.collected.info	4
http://127.0.0.1	3
http://staffmail.scu.edu.au	2
http://www.twylah.com	2
http://revolution-computing.typepad.com	2
http://www.hanrss.com	2
http://xianguo.com	2
http://webmail.scu.edu.au	2
http://cache.baidu.com	1

RHadoop, R meets Hadoop

by Revolution Analytics on Mar 06, 2012

Accessibility

Categories

Upload Details

Usage Rights

20 Embeds 34,343

Statistics

RHadoop, R meets Hadoop Presentation Transcript