In this talk, we will review an experience of rearchitecting and migrating a system that appeared reactive and microservice-based, but was in fact a monolith with RPC calls to a truly reactive architecture. The migration work had to be done without causing disruption to the current system, and without taking time to rewrite the system. The result is a biometric computer vision system with a distributed domain in Akka / Scala with storage in Apache Cassandra, with the computer vision components in OpenCV in C++, connected with RabbitMQ and with batch analytics code in Apache Spark. This talk will show the architectural and code smells that were the result of half-harted reactive implementation and the way to address them, but also the impact of the changes on privacy and security of the stored biometric information.

1. MANCHESTER LONDON NEW YORK
CC BY-NC 3.0

2. CC BY-NC 3.0
Screenshot

3. CC BY-NC 3.0
Before: the mess

4. CC BY-NC 3.0
Before: the mess

5. CC BY-NC 3.0
Before: the mess
Cassandra
Akka & Spray
C++ & CUDA
RabbitMQ

6. CC BY-NC 3.0
Before: the mess
scene topic
identity topic
(Map[String, String],
Array[Byte])
(Map[String, String],
Array[Byte])

7. CC BY-NC 3.0
Before: the mess
_tmp_x topics
JSON
scene topic
identity topic
(Map[String, String],
Array[Byte])
(Map[String, String],
Array[Byte])

8. CC BY-NC 3.0
Before: the mess
fire-and-forget at-most-once
non-durable
fire-and-forgetat-most-once

9. CC BY-NC 3.0
Asynchronous request–response orchestration
class Orchestrator extends Actor with ActorFSM[Orchestrator.State, Orchestrator.Data] {
startWith(Idle, UninitializedData)
when(Idle, idleTimeout)(idleSF)
when(ImageProcessing, stepTimeout)(imageProcessingSF)
when(WaitingForProcessingResult, stepTimeout)(waitingForProcessingSF)
whenUnhandled(timeoutSF)
onTransition {
case _ -> Aborted => ???
...
}
def idleSF: StateFunction = ???
def imageProcessingSF: StateFunction = ???
def waitingForProcessingSF: StateFunction = ???
def timeoutSF: StateFunction = {
case Event(StateTimeout, data: RunningTransactionData) => goto(Aborted)
}
}

10. CC BY-NC 3.0
After: proper microservices
scene
identity
ingest dashboard

11. CC BY-NC 3.0
After: proper microservices
scene
identity
ingest dashboard

12. CC BY-NC 3.0
After: proper microservices
scene
identity
ingest dashboard
Akka
Cassandra
Kafka

13. CC BY-NC 3.0
After: proper microservices
scene
identity
ingest dashboard
tweet-image topic
identity group
scene group
scene topic
identity topic

14. CC BY-NC 3.0
After: proper microservices
scene
identity
ingest dashboard
tweet-image topic
scene topic
identity topic
message Scene { … }
message Identity { … }
bytes image;
message Envelope {
int32 version = 1;
int64 processingTimestamp = 2;
int64 ingestionTimestamp = 3;
string correlationId = 4;
string messageId = 5;
string messageType = 6;
bytes payload = 7;
}

15. CC BY-NC 3.0
After: proper microservices
scene
identity
ingest dashboard
at-least-once I
at-least-once II
at-most-oncefire-and-forget

16. CC BY-NC 3.0
Persistence and formats
message Envelope {
int32 version = 1;
int64 processingTimestamp = 2;
int64 ingestionTimestamp = 3;
string correlationId = 4;
string messageId = 5;
string messageType = 6;
bytes payload = 7;
}

17. CC BY-NC 3.0
Persistence and formats
message Scene {
message Label {
string label = 1;
double score = 2;
}
repeated Label labels = 3;
}

18. CC BY-NC 3.0
Persistence and formats
message Identity {
oneof face {
IdentifiedFace identifiedFace = 1;
UnknownFace unknownFace = 2;
}
message IdentifiedFace {
string name = 1;
double score = 2;
}
message UnknownFace { }
}

19. CC BY-NC 3.0
Persistence and formats
message IdentifyFace {
int64 ingestionTimestamp = 2;
string correlationId = 3;
string handle = 4;
bytes image = 5;
}
message IdentifyFaces {
repeated IdentifyFace identifyFaces = 1;
}
message FaceImage {
double confidence = 1;
int32 x = 2;
int32 y = 3;
int32 w = 4;
int32 h = 5;
bytes rgbBitmap = 6;
}

20. CC BY-NC 3.0
Fire–and–forget send
object Act {
def props(config: Config): Props = {
val producerConf = KafkaProducer.Conf(config.getConfig("..."),
new StringSerializer, KafkaSerializer[Envelope](_.toByteArray))
Props(classOf[Act], producerConf)
}
}
class Act(producerConf: KafkaProducer.Conf[String, Envelope]) extends Actor {
private[this] val producer = KafkaProducer(conf = producerConf)
override def receive: Receive = {
case TweetImage(handle, content) =>
producer.send(KafkaProducerRecord("tweet-image", handle,
Envelope(version = 100,
ingestionTimestamp = System.nanoTime(),
processingTimestamp = System.nanoTime(),
messageId = UUID.randomUUID().toString,
correlationId = UUID.randomUUID().toString,
payload = content)))
}
}

21. CC BY-NC 3.0
At least once delivery I
class SceneClassifierActor(…) extends Actor {
private[this] val kafkaConsumerActor = context.actorOf(…)
private[this] val producer = KafkaProducer(…)
override def receive: Receive = {
case extractor(consumerRecords) =>
val futures = consumerRecords.pairs.flatMap {
case (Some(handle), envelope) =>
val outEnvelope = …
Some(producer.send(KafkaProducerRecord("scene", handle, outEnvelope)))
}
import context.dispatcher
Future.sequence(futures).onSuccess {
case _ => kafkaConsumerActor ! Confirm(consumerRecords.offsets, commit = true)
}
}
}

22. CC BY-NC 3.0
At least once delivery II
class IdentityMatcherActor(...) extends PersistentActor with AtLeastOnceDelivery {
override val persistenceId: String = "identity-matcher-actor"
def identifyFacesAndSend(identifyFaces: Seq[IdentifyFace])(implicit executor: ExecutionContext): Future[Unit] = {
// Future.sequence(producer.send(...))
}
def handleIdentifyFace: Receive = {
case (deliveryId: Long, identifyFaces: IdentifyFaces) =>
import context.dispatcher
identifyFacesAndSend(identifyFaces.identifyFaces).onSuccess { case _ => confirmDelivery(deliveryId) }
case IdentifyFaces(faces) =>
import context.dispatcher
identifyFacesAndSend(faces).onFailure { case _ => self ! Kill }
}
override def receiveRecover: Receive = handleIdentifyFace
override def receiveCommand: Receive = handleIdentifyFace orElse {
case extractor(consumerRecords) =>
val identifyFaces = consumerRecords.pairs.flatMap {
case (Some(handle), envelope) =>
Some(IdentifyFace(envelope.ingestionTimestamp, envelope.correlationId, handle, envelope.payload))
}
persist(IdentifyFaces(identifyFaces = identifyFaces)) { result =>
deliver(self.path)(deliveryId => (deliveryId, result))
sender() ! Confirm(consumerRecords.offsets, commit = true)
}
}
}

23. CC BY-NC 3.0
At most once delivery
class DashboardSinkActor(...) extends Actor {
private[this] val kafkaConsumerActor = context.actorOf(...)
override def receive: Receive = {
case extractor(consumerRecords) =>
consumerRecords.pairs.foreach {
case (None, _) =>
case (Some(handle), envelope) =>
context.system.eventStream.publish(
TweetEnvelope(version = 100,
handle = handle,
ingestionTimestamp = envelope.ingestionTimestamp,
messageId = envelope.messageId,
messageType = envelope.messageType,
payload = envelope.payload))
}
kafkaConsumerActor ! Confirm(consumerRecords.offsets, commit = true)
}
}

24. CC BY-NC 3.0
ENOUGH SLIDES!

25. CC BY-NC 3.0
After: proper microservices
scene
identity
ingest dashboard

26. MANCHESTER LONDON NEW YORK
CC BY-NC 3.0
github.com/eigengo/reactive-summit-2016
rsa16-models.s3-website-eu-west-1.amazonaws.com/{identity,scene}/{config,labels,params}
@honzam399
janm@cakesolutions.net

27. @cakesolutions
(347) 708-1518
enquiries@cakesolutions.net
MANCHESTER LONDON NEW YORK
CC BY-NC 3.0