Apache Samza*
Stream Processing at LinkedIn
Chris Riccomini
11/13/2013
* Incubating

Stream Processing?

0 ms
Response latency

0 ms
Response latency
Synchronous

0 ms
Response latency
Synchronous
Later. Possibly much later.

0 ms
Response latency
Milliseconds to minutes
Synchronous
Later. Possibly much later.

Newsfeed

News

Ad Relevance

Email

Search Indexing Pipeline

Metrics and Monitoring

Motivation

Real-time Feeds
•
•
•
•
User activity
Metrics
Monitoring
Database Changes

Real-time Feeds
• 10+ billion writes per day
• 172,000 messages per second (average)
• 55+ billion messages per day to real-time
consumers

Stream Processing is Hard
•
•
•
•
•
•
Partitioning
State
Re-processing
Failure semantics
Joins to services or database
Non-determinism

Samza Concepts
&
Architecture

Streams
Partition 0
Partition 1
Partition 2

Streams
Partition 0
1
2
3
4
5
6
Partition 1
1
2
3
4
5
Partition 2
1
2
3
4
5
6
7

Streams
Partition 0
1
2
3
4
5
6
Partition 1
1
2
3
4
5
Partition 2
1
2
3
4
5
6
7

Streams
Partition 0
1
2
3
4
5
6
Partition 1
1
2
3
4
5
Partition 2
1
2
3
4
5
6
7

Streams
Partition 0
1
2
3
4
5
6
Partition 1
1
2
3
4
5
Partition 2
1
2
3
4
5
6
7

Streams
Partition 0
1
2
3
4
5
6
Partition 1
1
2
3
4
5
Partition 2
1
2
3
4
5
6
7

Streams
Partition 0
1
2
3
4
5
6
Partition 1
1
2
3
4
5
Partition 2
1
2
3
4
5
6
7
next append

Tasks
Partition 0

Tasks
Partition 0
Task 1

Tasks
Partition 0
class PageKeyViewsCounterTask implements StreamTask {
public void process(IncomingMessageEnvelope envelope,
MessageCollector collector,
TaskCoordinator coordinator) {
GenericRecord record = ((GenericRecord) envelope.getMsg());
String pageKey = record.get("page-key").toString();
int newCount = pageKeyViews.get(pageKey).incrementAndGet();
collector.send(countStream, pageKey, newCount);
}
}

Tasks
Partition 0
class PageKeyViewsCounterTask implements StreamTask {
public void process(IncomingMessageEnvelope envelope,
MessageCollector collector,
TaskCoordinator coordinator) {
GenericRecord record = ((GenericRecord) envelope.getMsg());
String pageKey = record.get("page-key").toString();
int newCount = pageKeyViews.get(pageKey).incrementAndGet();
collector.send(countStream, pageKey, newCount);
}
}

Tasks
Partition 0
class PageKeyViewsCounterTask implements StreamTask {
public void process(IncomingMessageEnvelope envelope,
MessageCollector collector,
TaskCoordinator coordinator) {
GenericRecord record = ((GenericRecord) envelope.getMsg());
String pageKey = record.get("page-key").toString();
int newCount = pageKeyViews.get(pageKey).incrementAndGet();
collector.send(countStream, pageKey, newCount);
}
}

Tasks
Partition 0
class PageKeyViewsCounterTask implements StreamTask {
public void process(IncomingMessageEnvelope envelope,
MessageCollector collector,
TaskCoordinator coordinator) {
GenericRecord record = ((GenericRecord) envelope.getMsg());
String pageKey = record.get("page-key").toString();
int newCount = pageKeyViews.get(pageKey).incrementAndGet();
collector.send(countStream, pageKey, newCount);
}
}

Tasks
Partition 0
class PageKeyViewsCounterTask implements StreamTask {
public void process(IncomingMessageEnvelope envelope,
MessageCollector collector,
TaskCoordinator coordinator) {
GenericRecord record = ((GenericRecord) envelope.getMsg());
String pageKey = record.get("page-key").toString();
int newCount = pageKeyViews.get(pageKey).incrementAndGet();
collector.send(countStream, pageKey, newCount);
}
}

Tasks
Partition 0
class PageKeyViewsCounterTask implements StreamTask {
public void process(IncomingMessageEnvelope envelope,
MessageCollector collector,
TaskCoordinator coordinator) {
GenericRecord record = ((GenericRecord) envelope.getMsg());
String pageKey = record.get("page-key").toString();
int newCount = pageKeyViews.get(pageKey).incrementAndGet();
collector.send(countStream, pageKey, newCount);
}
}

Tasks
Partition 0
class PageKeyViewsCounterTask implements StreamTask {
public void process(IncomingMessageEnvelope envelope,
MessageCollector collector,
TaskCoordinator coordinator) {
GenericRecord record = ((GenericRecord) envelope.getMsg());
String pageKey = record.get("page-key").toString();
int newCount = pageKeyViews.get(pageKey).incrementAndGet();
collector.send(countStream, pageKey, newCount);
}
}

Tasks
Partition 0
class PageKeyViewsCounterTask implements StreamTask {
public void process(IncomingMessageEnvelope envelope,
MessageCollector collector,
TaskCoordinator coordinator) {
GenericRecord record = ((GenericRecord) envelope.getMsg());
String pageKey = record.get("page-key").toString();
int newCount = pageKeyViews.get(pageKey).incrementAndGet();
collector.send(countStream, pageKey, newCount);
}
}

Tasks
Partition 0
class PageKeyViewsCounterTask implements StreamTask {
public void process(IncomingMessageEnvelope envelope,
MessageCollector collector,
TaskCoordinator coordinator) {
GenericRecord record = ((GenericRecord) envelope.getMsg());
String pageKey = record.get("page-key").toString();
int newCount = pageKeyViews.get(pageKey).incrementAndGet();
collector.send(countStream, pageKey, newCount);
}
}

Tasks
Partition 0
class PageKeyViewsCounterTask implements StreamTask {
public void process(IncomingMessageEnvelope envelope,
MessageCollector collector,
TaskCoordinator coordinator) {
GenericRecord record = ((GenericRecord) envelope.getMsg());
String pageKey = record.get("page-key").toString();
int newCount = pageKeyViews.get(pageKey).incrementAndGet();
collector.send(countStream, pageKey, newCount);
}
}

Tasks
Partition 0
Task 1

Tasks
Page Views - Partition 0
1
2
3
4
PageKeyViews
CounterTask
Partition 0
Partition 1
Output Count Stream

Tasks
Page Views - Partition 0
1
2
3
4
PageKeyViews
CounterTask
Partition 0
Partition 1
Output Count Stream

Tasks
Page Views - Partition 0
1
2
3
4
PageKeyViews
CounterTask
Partition 0
Partition 1
Output Count Stream

Tasks
Page Views - Partition 0
1
2
3
4
PageKeyViews
CounterTask
Output Count Stream
Partition 0
Partition 1

Tasks
Page Views - Partition 0
1
2
3
4
PageKeyViews
CounterTask
Output Count Stream
Partition 0
Partition 1

Tasks
Page Views - Partition 0
1
2
3
4
PageKeyViews
CounterTask
Output Count Stream
Partition 0
Partition 1

Tasks
Page Views - Partition 0
1
2
3
4
PageKeyViews
CounterTask
Output Count Stream
Partition 0
Partition 1

Tasks
Page Views - Partition 0
1
2
3
4
PageKeyViews
CounterTask
Output Count Stream
Partition 0
Partition 1

Tasks
Page Views - Partition 0
1
2
3
4
PageKeyViews
CounterTask
Checkpoint
Stream
2
Output Count Stream
Partition 1
Partition 0
Partition 1

Tasks
Page Views - Partition 0
1
2
3
4
PageKeyViews
CounterTask
Checkpoint
Stream
2
Output Count Stream
Partition 1
Partition 0
Partition 1

Tasks
Page Views - Partition 0
1
2
3
4
PageKeyViews
CounterTask
Checkpoint
Stream
2
Output Count Stream
Partition 1
Partition 0
Partition 1

Tasks
Page Views - Partition 0
1
2
3
4
PageKeyViews
CounterTask
Checkpoint
Stream
2
Output Count Stream
Partition 1
Partition 0
Partition 1

Tasks
Page Views - Partition 0
1
2
3
4
PageKeyViews
CounterTask
Checkpoint
Stream
2
Output Count Stream
Partition 1
Partition 0
Partition 1

Tasks
Page Views - Partition 0
1
2
3
4
PageKeyViews
CounterTask
Checkpoint
Stream
2
Output Count Stream
Partition 1
Partition 0
Partition 1

Tasks
Page Views - Partition 0
1
2
3
4
PageKeyViews
CounterTask
Checkpoint
Stream
2
Output Count Stream
Partition 1
Partition 0
Partition 1

Tasks
Page Views - Partition 0
1
2
3
4
PageKeyViews
CounterTask
Checkpoint
Stream
2
Output Count Stream
Partition 1
Partition 0
Partition 1

Tasks
Page Views - Partition 0
1
2
3
4
PageKeyViews
CounterTask
Checkpoint
Stream
2
Output Count Stream
Partition 1
Partition 0
Partition 1

Jobs
Stream A
Task 1
Task 2
Stream B
Task 3

Jobs
Stream A
Task 1
Stream B
Task 2
Stream C
Task 3

Jobs
AdViews
Task 1
AdClicks
Task 2
AdClickThroughRate
Task 3

Jobs
AdViews
Task 1
AdClicks
Task 2
AdClickThroughRate
Task 3

Jobs
Stream A
Task 1
Stream B
Task 2
Stream C
Task 3

Dataflow
Stream A
Stream B
Job 1
Stream D
Job 2
Stream E
Job 3
Stream B
Stream C

Dataflow
Stream A
Stream B
Job 1
Stream D
Job 2
Stream E
Job 3
Stream B
Stream C

YARN

YARN
You: I want to run command X on two machines with
512M of memory.

YARN
You: I want to run command X on two machines with
512M of memory.
YARN: Cool, where’s your code?

YARN
You: I want to run command X on two machines with
512M of memory.
YARN: Cool, where’s your code?
You: http://some-host/jobs/download/my.tgz

YARN
You: I want to run command X on two machines with
512M of memory.
YARN: Cool, where’s your code?
You: http://some-host/jobs/download/my.tgz
YARN: I’ve run your command on grid-node-2 and
grid-node-7.

YARN
Host 1
Host 2
Host 3

YARN
Host 1
Host 2
Host 3
NM
NM
NM

YARN
Host 0
RM
Host 1
Host 2
Host 3
NM
NM
NM

YARN
Host 0
Client
RM
Host 1
Host 2
Host 3
NM
NM
NM

YARN
Host 0
Client
RM
Host 1
Host 2
Host 3
NM
NM
NM

YARN
Host 0
Client
RM
Host 1
Host 2
Host 3
NM
NM
NM

YARN
Host 0
Client
Host 1
NM
RM
Host 2
AM
Host 3
NM
NM

YARN
Host 0
Client
Host 1
NM
RM
Host 2
AM
Host 3
NM
NM

YARN
Host 0
Client
Host 1
NM
RM
Host 2
AM
Host 3
NM
NM

YARN
Host 0
Client
Host 1
NM
RM
Host 2
AM
Host 3
NM
NM
Container

YARN
Host 0
Client
Host 1
NM
RM
Host 2
AM
Host 3
NM
NM
Container

YARN
Host 0
Client
Host 1
NM
RM
Host 2
AM
Host 3
NM
NM

YARN
Host 0
Client
Host 1
NM
RM
Host 2
AM
Host 3
NM
NM

YARN
Host 0
Client
Host 1
NM
RM
Host 2
AM
Host 3
NM
NM

YARN
Host 0
Client
Host 1
NM
RM
Host 2
AM
Host 3
NM
NM

YARN
Host 0
Client
Host 1
NM
RM
Host 2
AM
Host 3
NM
NM
Container

Jobs
Stream A
Task 1
Task 2
Stream B
Task 3

Containers
Stream A
Task 1
Task 2
Stream B
Task 3

Containers
Stream A
Samza Container 1
Stream B
Samza Container 2

Containers
Samza Container 1
Samza Container 2

YARN
Host 1
Samza Container 1
Host 2
Samza Container 2

YARN
Host 1
Host 2
NodeManager
NodeManager
Samza Container 1
Samza Container 2

YARN
Host 1
Host 2
NodeManager
NodeManager
Samza Container 1
Samza Container 2
Samza YARN AM

YARN
Host 1
Host 2
NodeManager
NodeManager
Samza Container 1
Kafka Broker
Samza Container 2
Samza YARN AM
Kafka Broker

YARN
Host 1
Host 2
NodeManager
NodeManager
MapReduce
Container
HDFS
MapReduce
YARN AM
MapReduce
Container
HDFS

YARN
Host 1
Stream A
NodeManager
Samza Container 1
Samza Container 1
Kafka Broker
Stream C
Samza
Container 2

YARN
Host 1
Stream A
NodeManager
Samza Container 1
Samza Container 1
Kafka Broker
Stream C
Samza
Container 2

YARN
Host 1
Stream A
NodeManager
Samza Container 1
Samza Container 1
Kafka Broker
Stream C
Samza
Container 2

YARN
Host 1
Stream A
NodeManager
Samza Container 1
Samza Container 1
Kafka Broker
Stream C
Samza
Container 2

YARN
Host 1
Host 2
NodeManager
NodeManager
Samza Container 1
Kafka Broker
Samza Container 2
Samza YARN AM
Kafka Broker

CGroups
Host 1
Host 2
NodeManager
NodeManager
Samza Container 1
Kafka Broker
Samza Container 2
Samza YARN AM
Kafka Broker

(Not Running) Multi-Framework
Host 1
Host 2
NodeManager
NodeManager
Samza Container 1
Kafka
MapReduce
Container
Samza YARN AM
HDFS

Stateful Processing

SELECT
col1,
count(*)
FROM
stream1
INNER JOIN
stream2
ON
stream1.col3 = stream2.col3
WHERE
col2 > 20
GROUP BY
col1
ORDER BY
count(*) DESC
LIMIT 50;

SELECT
col1,
count(*)
FROM
stream1
INNER JOIN
stream2
ON
stream1.col3 = stream2.col3
WHERE
col2 > 20
GROUP BY
col1
ORDER BY
count(*) DESC
LIMIT 50;

SELECT
col1,
count(*)
FROM
stream1
INNER JOIN
stream2
ON
stream1.col3 = stream2.col3
WHERE
col2 > 20
GROUP BY
col1
ORDER BY
count(*) DESC
LIMIT 50;

SELECT
col1,
count(*)
FROM
stream1
INNER JOIN
stream2
ON
stream1.col3 = stream2.col3
WHERE
col2 > 20
GROUP BY
col1
ORDER BY
count(*) DESC
LIMIT 10;

How do people do this?

Remote Stores
Stream A
Task 1
Task 2
Task 3
Key-Value
Store
Stream B

Remote RPC is slow
• Stream: ~500k records/sec/container
• DB: << less

Online vs. Async

No undo
• Database state is non-deterministic
• Can’t roll back mutations if task crashes

Tables & Streams
put(a, w)
put(b, x)
Database
put(a, y)
put(b, z)
Time

Stateful Tasks
Stream A
Task 1
Task 2
Stream B
Task 3

Stateful Tasks
Stream A
Task 1
Task 2
Stream B
Task 3

Stateful Tasks
Stream A
Task 1
Task 2
Stream B
Task 3
Changelog Stream

Stateful Tasks
Stream A
Task 1
Task 2
Stream B
Task 3
Changelog Stream

Stateful Tasks
Stream A
Task 1
Task 2
Stream B
Task 3
Changelog Stream

Stateful Tasks
Stream A
Task 1
Task 2
Stream B
Task 3
Changelog Stream

Stateful Tasks
Stream A
Task 1
Task 2
Stream B
Task 3
Changelog Stream

Stateful Tasks
Stream A
Task 1
Task 2
Stream B
Task 3
Changelog Stream

Stateful Tasks
Stream A
Task 1
Task 2
Stream B
Task 3
Changelog Stream

Stateful Tasks
Stream A
Task 1
Task 2
Stream B
Task 3
Changelog Stream

Stateful Tasks
Stream A
Task 1
Task 2
Stream B
Task 3
Changelog Stream

Stateful Tasks
Stream A
Task 1
Task 2
Stream B
Task 3
Changelog Stream

Stateful Tasks
Stream A
Task 1
Task 2
Stream B
Task 3
Changelog Stream

Stateful Tasks
Stream A
Task 1
Task 2
Stream B
Task 3
Changelog Stream

Key-Value Store
•
•
•
•
put(table_name, key, value)
get(table_name, key)
delete(table_name, key)
range(table_name, key1, key2)

Stateful Stream Task
public class SimpleStatefulTask implements StreamTask, InitableTask {
private KeyValueStore<String, String> store;
public void init(Config config, TaskContext context) {
this.store = context.getStore("mystore");
}
public void process(
IncomingMessageEnvelope envelope,
MessageCollector collector,
TaskCoordinator coordinator) {
GenericRecord record = (GenericRecord) envelope.getMessage();
String memberId = record.get("member_id");
String name = record.get("name");
System.out.println("old name: " + store.get(memberId));
store.put(memberId, name);
}
}

Stateful Stream Task
public class SimpleStatefulTask implements StreamTask, InitableTask {
private KeyValueStore<String, String> store;
public void init(Config config, TaskContext context) {
this.store = context.getStore("mystore");
}
public void process(
IncomingMessageEnvelope envelope,
MessageCollector collector,
TaskCoordinator coordinator) {
GenericRecord record = (GenericRecord) envelope.getMessage();
String memberId = record.get("member_id");
String name = record.get("name");
System.out.println("old name: " + store.get(memberId));
store.put(memberId, name);
}
}

Stateful Stream Task
public class SimpleStatefulTask implements StreamTask, InitableTask {
private KeyValueStore<String, String> store;
public void init(Config config, TaskContext context) {
this.store = context.getStore("mystore");
}
public void process(
IncomingMessageEnvelope envelope,
MessageCollector collector,
TaskCoordinator coordinator) {
GenericRecord record = (GenericRecord) envelope.getMessage();
String memberId = record.get("member_id");
String name = record.get("name");
System.out.println("old name: " + store.get(memberId));
store.put(memberId, name);
}
}

Stateful Stream Task
public class SimpleStatefulTask implements StreamTask, InitableTask {
private KeyValueStore<String, String> store;
public void init(Config config, TaskContext context) {
this.store = context.getStore("mystore");
}
public void process(
IncomingMessageEnvelope envelope,
MessageCollector collector,
TaskCoordinator coordinator) {
GenericRecord record = (GenericRecord) envelope.getMessage();
String memberId = record.get("member_id");
String name = record.get("name");
System.out.println("old name: " + store.get(memberId));
store.put(memberId, name);
}
}

Whew!

Let’s be Friends!
• We are incubating, and you can help!
• Get up and running in 5 minutes
http://bit.ly/hello-samza
• Grab some newbie JIRAs
http://bit.ly/samza_newbie_issues