The document discusses Scala concepts like implicit conversions and parameters as well as data processing frameworks like Hadoop, Cascading, and Scalding. It provides examples of using these concepts and frameworks to count the frequency of the first characters in words from a file using different approaches, from basic Scala to implementations with Cascading and Scalding.

1. TUPLESTUPLES
ALLALL
THE WAYTHE WAY
DOWNDOWN

2. HIHI

5. ABOUT THIS TALKABOUT THIS TALK
Scala
Hadoop
Cascading
Scalding

6. SCALASCALA

7. IMPLICITIMPLICIT
CONVERSIONSCONVERSIONS

8. scala> "blah and blah"
res4: java.lang.String = blah and blah
scala> "blah and blah".urlencode()
error: value urlencode is not a member of java.lang.String

9. class WrappedString(val x:String) {
def urlencode() = URLEncoder.encode(x)
}

10. implicit def stringToWrappedString(x:String): WrappedString =
new WrappedString(x)

11. scala> "blah and blah".urlencode()
res2: java.lang.String = blah+and+blah!
scala> stringToWrappedString(“blah and blah”).urlencode()
res2: java.lang.String = blah+and+blah!

12. IMPLICIT PARAMETERSIMPLICIT PARAMETERS

13. def emailSignature(name:String)(implicit organization:String) =
"Sincerely Yours,n" + name + "n" + organization
scala> implicit val organization = "Etsy, Inc."
organization: java.lang.String = Etsy, Inc.
scala> emailSignature("Giovanni Fernandez-Kincade")
res3: java.lang.String =
Sincerely Yours,
Giovanni Fernandez-Kincade
Etsy, Inc.

14. WORD COUNTWORD COUNT
FIRST CHARACTERFIRST CHARACTER
COUNTCOUNT

15. val words = List("one", "two", "three")
words
.map {
word =>
Map(word.substring(0,1) -> 1)
}
.reduce {
(a, b) =>
a ++ b.map {
mapTuple =>
val (key, count) = mapTuple
(key, a.getOrElse(key, 0) + count)
}
}

16. val words = List("one", "two", "three")
words
.map {
word =>
Map(word.substring(0,1) -> 1)
}
List(Map(o -> 1), Map(t -> 1), Map(t -> 1))

17. List(Map(o -> 1), Map(t -> 1), Map(t -> 1))
.reduce {
(a, b) =>
a ++ b.map {
mapTuple =>
val (key, count) = mapTuple
(key, a.getOrElse(key, 0) + count)
}
}
Map(o -> 1, t -> 2)

18. HADOOPHADOOP

19. ~> echo "onentwonthree" > ~/foo.txt
~> hadoop fs -put ~/foo.txt

22. SORT OFSORT OF
void map(K1 key, V1 value, OutputCollector<K2,V2> output)
void reduce(K2 key, Iterator<V2> values,
OutputCollector<K3,V3> output)

23. CASCADINGCASCADING

24. cascading.tuple.Tuple

25. val tuple = new Tuple(
1.asInstanceOf[Object],
"gigi".asInstanceOf[Object],
"miami".asInstanceOf[Object]
)
val fields = new Fields(
"user_id",
"user_name",
"location"
)
val one = new TupleEntry(
fields,
tuple
)

26. > one.get("user_id")
res4: java.lang.Comparable[_] = 1
> one.get("user_name")
res5: java.lang.Comparable[_] = gigi

27. FIRST CHARACTERFIRST CHARACTER
COUNT 2.0COUNT 2.0

29. def cascadingTuple[T](fieldName:String, entry:T): TupleEntry =
new TupleEntry(
new Fields(fieldName),
new Tuple(entry.asInstanceOf[Object])
)

30. val words = List(
cascadingTuple("word", "one"),
cascadingTuple("word", "two"),
cascadingTuple("word", "three")
)

31. words
.map {
tuple =>
cascadingTuple(
"map",
Map(tuple.getString("word").substring(0,1) -> 1)
)
}

32. .reduce {
(tupleA, tupleB) =>
val (a,b) = (
tupleA.getObject("map").asInstanceOf[Map[String,Int]],
tupleB.getObject("map").asInstanceOf[Map[String,Int]],
)
val result = a ++ b.map {
mapTuple =>
val (key, count) = mapTuple
(key, a.getOrElse(key, 0) + count)
}
cascadingTuple("map", result)
}

33. > run
fields: ['map'] tuple: ['Map(o -> 1, t -> 2)']

34. FIRST CHARACTERFIRST CHARACTER
COUNT 1.2COUNT 1.2

35. List("one", "two", "three")
.groupBy(_.substring(0,1))
res1: Map(o -> List(one), t -> List(two, three))

36. List("one", "two", "three")
.groupBy(_.substring(0,1))
.map {
(tuple) =>
val (key, value) = tuple
(key, value.size)
}
res1: Map(o -> 1, t -> 2)

37. SCALDINGSCALDING

38. import com.twitter.scalding._
class ExampleJob(args: Args)
extends Job(args) {
TextLine("data/words.txt")
.map('line -> 'first_character) {
(line:String) =>
line.substring(0,1)
}
.groupBy('first_character) {
(g: GroupBuilder) =>
g.size('size)
}
.write(Tsv("data/output/characters.tsv"))
}

39. class ExampleJob(args: Args)
extends Job(args) {

40. TextLine("data/words.txt")
fields: ['line'] tuple: ['one']
fields: ['line'] tuple: ['two']
fields: ['line'] tuple: ['three']

41. .map('line -> 'first_character) {
(line:String) =>
line.substring(0,1)
}
fields: ['line', 'first_character'] tuple: ['one', 'o']
fields: ['line', 'first_character'] tuple: ['two', 't']
fields: ['line', 'first_character'] tuple: ['three', 't']

42. .groupBy('first_character) {
(g: GroupBuilder) =>
g.size('size)
}
fields: ['first_character', 'size'] tuple: ['o', 1]
fields: ['first_character', 'size'] tuple: ['t', 2]

43. .groupBy('first_character) {
(g: GroupBuilder) =>
g.size('size)
}
fields: ['first_character', ‘size’] tuple: ['o', ‘1’]
fields: ['first_character', ‘size’] tuple: ['t', ‘2’]
.write(Tsv("data/output/characters.tsv"))
o 1
t 2

44. BEHIND THE SCENESBEHIND THE SCENES

45. TextLine("data/words.txt")
.map('line -> 'first_character) {

46. HUH?HUH?

48. class Job(val args: Args)
extends FieldConversions
with java.io.Serializable {
implicit def pipeToRichPipe(pipe: Pipe): RichPipe =
new RichPipe(pipe)
implicit def sourceToRichPipe(src: Source): RichPipe =
new RichPipe(src.read)

49. def map[A, T](fs: (Fields, Fields))
(fn: A => T)
(implicit conv: TupleConverter[A],
setter: TupleSetter[T]): Pipe

50. .map('line -> 'first_character) {
def map[A, T](fs: (Fields, Fields))
implicit def symbolToFields(x: Symbol) = {
if (x == '*) {
Fields.ALL
} else {
new Fields(x.name)
}
}

51. .map('line -> 'first_character) {
(line:String) =>
line.substring(0,1)
}
def map[A, T](fs: (Fields, Fields))
(fn: A => T)

52. implicit def tuple1Converter[A](implicit gA: TupleGetter[A]): TupleCo
implicit object StringGetter extends TupleGetter[String] {
override def get(tup: CTuple, i: Int) = tup.getString(i)
}
.map('line -> 'first_character) {
(line:String) =>
line.substring(0,1)
}
def map[A, T](fs: (Fields, Fields))
(fn: A => T)
(implicit conv: TupleConverter[A],
setter: TupleSetter[T]): Pipe
implicit def singleSetter[A]: TupleSetter[A] = new TupleSetter[A] {
override def apply(arg: A) = {
val tup = CTuple.size(1)
tup.set(0, arg)
tup
}

53. WHAT HAPPENS WHENWHAT HAPPENS WHEN
YOU RUN IT?YOU RUN IT?
> runMain com.twitter.scalding.Tool
com.giokincade.scalding.ExampleJob --local
hadoop jar foo.jar com.giokincade.scalding.ExampleJob --hdfs

54. COMPILE TIMECOMPILE TIME

55. COMPOSITION TIMECOMPOSITION TIME

58. JOB RUN TIMEJOB RUN TIME

59. TYPED PIPETYPED PIPE

60. TypedPipe.from(
TextLine("data/words.txt")
)
.map(line => line.substring(0,1))
.groupBy(character => character)
.size
.write(
TypedTsv[(String, Long)]("data/output/typed-characters.tsv")
)

61. RESOURCESRESOURCES
github.com/giokincade/scalding-talk-examples
github.com/twitter/scalding

62. MEME
gio@etsy.com
@giokincade

63. FINFIN