![Data
Model
Training
Model
Serving
Raw
Data
Model
Params
Scored
Records
val builder = new StreamsBuilderS // New Scala Wrapper API.
val data = builder.stream[Array[Byte], Array[Byte]](rawDataTopic)
val model = builder.stream[Array[Byte], Array[Byte]](modelTopic)
val modelProcessor = new ModelProcessor
val scorer = new Scorer(modelProcessor) // scorer.score(record) used
model.mapValues(bytes => Model.parseBytes(bytes)) // array => record
.filter((key, model) => model.valid) // Successful?
.mapValues(model => ModelImpl.findModel(model))
.process(() => modelProcessor, …) // Set up actual model
data.mapValues(bytes => DataRecord.parseBytes(bytes))
.filter((key, record) => record.valid)
.mapValues(record => new ScoredRecord(scorer.score(record),record))
.to(scoredRecordsTopic)
val streams = new KafkaStreams(
builder.build, streamsConfiguration)
streams.start()
sys.addShutdownHook(streams.close())](https://image.slidesharecdn.com/081550lightbend-181116142428/85/Big-Data-LDN-2018-STREAMING-DATA-MICROSERVICES-WITH-AKKA-STREAMS-KAFKA-STREAMS-AND-KAFKA-33-320.jpg)
![val builder = new StreamsBuilderS // New Scala Wrapper API.
val data = builder.stream[Array[Byte], Array[Byte]](rawDataTopic)
val model = builder.stream[Array[Byte], Array[Byte]](modelTopic)
val modelProcessor = new ModelProcessor
val scorer = new Scorer(modelProcessor) // scorer.score(record) used
model.mapValues(bytes => Model.parseBytes(bytes)) // array => record
.filter((key, model) => model.valid) // Successful?
.mapValues(model => ModelImpl.findModel(model))
.process(() => modelProcessor, …) // Set up actual model
data.mapValues(bytes => DataRecord.parseBytes(bytes))
.filter((key, record) => record.valid)
.mapValues(record => new ScoredRecord(scorer.score(record),record))
.to(scoredRecordsTopic)
val streams = new KafkaStreams(
builder.build, streamsConfiguration)
streams.start()
sys.addShutdownHook(streams.close())
Data
Model
Training
Model
Serving
Raw
Data
Model
Params
Scored
Records](https://image.slidesharecdn.com/081550lightbend-181116142428/85/Big-Data-LDN-2018-STREAMING-DATA-MICROSERVICES-WITH-AKKA-STREAMS-KAFKA-STREAMS-AND-KAFKA-34-320.jpg)
![val builder = new StreamsBuilderS // New Scala Wrapper API.
val data = builder.stream[Array[Byte], Array[Byte]](rawDataTopic)
val model = builder.stream[Array[Byte], Array[Byte]](modelTopic)
val modelProcessor = new ModelProcessor
val scorer = new Scorer(modelProcessor) // scorer.score(record) used
model.mapValues(bytes => Model.parseBytes(bytes)) // array => record
.filter((key, model) => model.valid) // Successful?
.mapValues(model => ModelImpl.findModel(model))
.process(() => modelProcessor, …) // Set up actual model
data.mapValues(bytes => DataRecord.parseBytes(bytes))
.filter((key, record) => record.valid)
.mapValues(record => new ScoredRecord(scorer.score(record),record))
.to(scoredRecordsTopic)
val streams = new KafkaStreams(
builder.build, streamsConfiguration)
streams.start()
sys.addShutdownHook(streams.close())
Data
Model
Training
Model
Serving
Raw
Data
Model
Params
Scored
Records](https://image.slidesharecdn.com/081550lightbend-181116142428/85/Big-Data-LDN-2018-STREAMING-DATA-MICROSERVICES-WITH-AKKA-STREAMS-KAFKA-STREAMS-AND-KAFKA-35-320.jpg)
![val builder = new StreamsBuilderS // New Scala Wrapper API.
val data = builder.stream[Array[Byte], Array[Byte]](rawDataTopic)
val model = builder.stream[Array[Byte], Array[Byte]](modelTopic)
val modelProcessor = new ModelProcessor
val scorer = new Scorer(modelProcessor) // scorer.score(record) used
model.mapValues(bytes => Model.parseBytes(bytes)) // array => record
.filter((key, model) => model.valid) // Successful?
.mapValues(model => ModelImpl.findModel(model))
.process(() => modelProcessor, …) // Set up actual model
data.mapValues(bytes => DataRecord.parseBytes(bytes))
.filter((key, record) => record.valid)
.mapValues(record => new ScoredRecord(scorer.score(record),record))
.to(scoredRecordsTopic)
val streams = new KafkaStreams(
builder.build, streamsConfiguration)
streams.start()
sys.addShutdownHook(streams.close())
Data
Model
Training
Model
Serving
Raw
Data
Model
Params
Scored
Records](https://image.slidesharecdn.com/081550lightbend-181116142428/85/Big-Data-LDN-2018-STREAMING-DATA-MICROSERVICES-WITH-AKKA-STREAMS-KAFKA-STREAMS-AND-KAFKA-36-320.jpg)
![Akka Streams
• The “gist” - calculate factorials:
val source: Source[Int, NotUsed] = Source(1 to 10)
val factorials = source.scan(BigInt(1)) (
(total, next) => total * next )
factorials.runWith(Sink.foreach(println))
1
2
6
24
120
720
5040
40320
362880
3628800](https://image.slidesharecdn.com/081550lightbend-181116142428/85/Big-Data-LDN-2018-STREAMING-DATA-MICROSERVICES-WITH-AKKA-STREAMS-KAFKA-STREAMS-AND-KAFKA-42-320.jpg)
![val source: Source[Int, NotUsed] = Source(1 to 10)
val factorials = source.scan(BigInt(1)) (
(total, next) => total * next )
factorials.runWith(Sink.foreach(println))
Akka Streams
• The “gist” - calculate factorials:
Source Flow Sink
A “Graph”
1
2
6
24
120
720
5040
40320
362880
3628800](https://image.slidesharecdn.com/081550lightbend-181116142428/85/Big-Data-LDN-2018-STREAMING-DATA-MICROSERVICES-WITH-AKKA-STREAMS-KAFKA-STREAMS-AND-KAFKA-43-320.jpg)
![Akka Cluster
Data
Model
Training
Model
Serving
Raw
Data
Model
Params
Alpakka
implicit val system = ActorSystem("ModelServing")
implicit val materializer = ActorMaterializer()
implicit val executionContext = system.dispatcher
val modelProcessor = new ModelProcessor // Same as KS example
val scorer = new Scorer(modelProcessor) // Same as KS example
val modelScoringStage = new ModelScoringStage(scorer)// AS custom “stage”
val dataStream: Source[Record, Consumer.Control] =
Consumer.atMostOnceSource(dataConsumerSettings,
Subscriptions.topics(rawDataTopic))
.map(input => DataRecord.parseBytes(input.value()))
.collect{ case Success(data) => data }
val modelStream: Source[ModelImpl, Consumer.Control] =
Consumer.atMostOnceSource(modelConsumerSettings,
Subscriptions.topics(modelTopic))
.map(input => Model.parseBytes(input.value()))
.collect{ case Success(mod) => mod }
.map(model => ModelImpl.findModel(model))](https://image.slidesharecdn.com/081550lightbend-181116142428/85/Big-Data-LDN-2018-STREAMING-DATA-MICROSERVICES-WITH-AKKA-STREAMS-KAFKA-STREAMS-AND-KAFKA-44-320.jpg)
![Akka Cluster
Data
Model
Training
Model
Serving
Raw
Data
Model
Params
Alpakka
implicit val system = ActorSystem("ModelServing")
implicit val materializer = ActorMaterializer()
implicit val executionContext = system.dispatcher
val modelProcessor = new ModelProcessor // Same as KS example
val scorer = new Scorer(modelProcessor) // Same as KS example
val modelScoringStage = new ModelScoringStage(scorer)// AS custom “stage”
val dataStream: Source[Record, Consumer.Control] =
Consumer.atMostOnceSource(dataConsumerSettings,
Subscriptions.topics(rawDataTopic))
.map(input => DataRecord.parseBytes(input.value()))
.collect{ case Success(data) => data }
val modelStream: Source[ModelImpl, Consumer.Control] =
Consumer.atMostOnceSource(modelConsumerSettings,
Subscriptions.topics(modelTopic))
.map(input => Model.parseBytes(input.value()))
.collect{ case Success(mod) => mod }
.map(model => ModelImpl.findModel(model))](https://image.slidesharecdn.com/081550lightbend-181116142428/85/Big-Data-LDN-2018-STREAMING-DATA-MICROSERVICES-WITH-AKKA-STREAMS-KAFKA-STREAMS-AND-KAFKA-45-320.jpg)
![Akka Cluster
Data
Model
Training
Model
Serving
Raw
Data
Model
Params
Alpakka
implicit val system = ActorSystem("ModelServing")
implicit val materializer = ActorMaterializer()
implicit val executionContext = system.dispatcher
val modelProcessor = new ModelProcessor // Same as KS example
val scorer = new Scorer(modelProcessor) // Same as KS example
val modelScoringStage = new ModelScoringStage(scorer)// AS custom “stage”
val dataStream: Source[Record, Consumer.Control] =
Consumer.atMostOnceSource(dataConsumerSettings,
Subscriptions.topics(rawDataTopic))
.map(input => DataRecord.parseBytes(input.value()))
.collect{ case Success(data) => data }
val modelStream: Source[ModelImpl, Consumer.Control] =
Consumer.atMostOnceSource(modelConsumerSettings,
Subscriptions.topics(modelTopic))
.map(input => Model.parseBytes(input.value()))
.collect{ case Success(mod) => mod }
.map(model => ModelImpl.findModel(model))](https://image.slidesharecdn.com/081550lightbend-181116142428/85/Big-Data-LDN-2018-STREAMING-DATA-MICROSERVICES-WITH-AKKA-STREAMS-KAFKA-STREAMS-AND-KAFKA-46-320.jpg)
![Akka Cluster
Data
Model
Training
Model
Serving
Raw
Data
Model
Params
Alpakka
.collect{ case Success(data) => data }
val modelStream: Source[ModelImpl, Consumer.Control] =
Consumer.atMostOnceSource(modelConsumerSettings,
Subscriptions.topics(modelTopic))
.map(input => Model.parseBytes(input.value()))
.collect{ case Success(mod) => mod }
.map(model => ModelImpl.findModel(model))
.collect{ case Success(modImpl) => modImpl }
.foreach(modImpl => modelProcessor.setModel(modImpl))
modelStream.to(Sink.ignore).run() // No “sinking” required; just run
dataStream
.viaMat(modelScoringStage)(Keep.right)
.map(result => new ProducerRecord[Array[Byte], ScoredRecord](
scoredRecordsTopic, result))
.runWith(Producer.plainSink(producerSettings))](https://image.slidesharecdn.com/081550lightbend-181116142428/85/Big-Data-LDN-2018-STREAMING-DATA-MICROSERVICES-WITH-AKKA-STREAMS-KAFKA-STREAMS-AND-KAFKA-47-320.jpg)
Big Data LDN 2018: STREAMING DATA MICROSERVICES WITH AKKA STREAMS, KAFKA STREAMS, AND KAFKA
- 1. Dean Wampler, Ph.D. [email protected] @deanwampler Streaming Microservices With Akka Streams and Kafka Streams
- 6. Streaming architectures (from the report)
- 7. Kubernetes, Mesos, YARN, … Cloud or on-premise Files Sockets REST ZooKeeper Cluster ZK Mini-batch Spark Batch Spark … Low Latency Flink Ka5a Streams Akka Streams Beam Persistence S3, … HDFS DiskDiskDisk SQL/ NoSQL Search 1 5 6 3 10 KaFa Cluster Broker 2 4 7 8 9 Beam Spark Events Streams Storage Microservices ReacBve PlaEorm Go Node.js …
- 8. Kubernetes, Mesos, YARN, … Cloud or on-premise Files Sockets REST ZooKeeper Cluster ZK Mini-batch Spark Batch Spark … Low Latency Flink Ka5a Streams Akka Streams Beam Persistence S3, … HDFS DiskDiskDisk SQL/ NoSQL Search 1 5 6 3 10 KaFa Cluster Broker 2 4 7 8 9 Beam Spark Events Streams Storage Microservices ReacBve PlaEorm Go Node.js … Today’s focus: •Kafka - the data backplane •Akka Streams and Kafka Streams - streaming microservices
- 9. Why Kafka?
- 10. Files Sockets REST ZooKeeper Cluster ZK Ka5 Ak S3, … HDFS DiskDiskDisk 1 5 6 3 10 KaFa Cluster Broker 2 4 7 8 Events Streams Microservices ReacBve PlaEorm Go Node.js … Why Kafka?
- 11. Organized into topics Ka#a Partition 1 Partition 2 Topic A Partition 1Topic B Topics are partitioned, replicated, and distributed
- 12. Unlike queues, consumers don’t delete entries; Kafka manages their lifecycles M Producers N Consumers, who start reading where they want Consumer 1 (at offset 14) 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 Partition 1Topic B Producer 1 Producer 2 Consumer 2 (at offset 10) writes reads Consumer 3 (at offset 6) earliest latest Logs, not queues!
- 13. Service 1 Log & Other Files Internet Services Service 2 Service 3 Services Services N * M links ConsumersProducers Before: Kafka for Connectivity X
- 14. Service 1 Log & Other Files Internet Services Service 2 Service 3 Services Services N * M links ConsumersProducers Before: Kafka for Connectivity Service 1 Log & Other Files Internet Services Service 2 Service 3 Services Services N + M links ConsumersProducers After: X X
- 15. Kafka for Connectivity Service 1 Log & Other Files Internet Services Service 2 Service 3 Services Services N + M links ConsumersProducers After: • Simplify dependencies • Resilient against data loss • M producers, N consumers • Simplicity of one “API” for communication
- 17. Files Sockets REST ZooKeeper Cluster ZK Mini-batch Spark Batch Spark … Low Latency Flink Ka5a Streams Akka Streams Beam Persistence S3, … HDFS DiskDiskDisk SQL/ NoSQL Search 1 5 6 KaFa Cluster Broker 2 4 7 8 9 Beam Spark Events Streams Storage Microservices Go Node.js … Spark, Flink - services to which you submit work. Large scale, automatic data partitioning. Beam - similar. Google’s project that has been instrumental in defining streaming semantics.
- 18. Files Sockets REST ZooKeeper Cluster ZK Mini-batch Spark Batch Spark … Low Latency Flink Ka5a Streams Akka Streams Beam Persistence S3, … HDFS DiskDiskDisk SQL/ NoSQL Search 1 5 6 KaFa Cluster Broker 2 4 7 8 9 Beam Spark Events Streams Storage Microservices Go Node.js … They do a lot (Spark example) …NodeNode Spark Driver object MyApp { def main() { val ss = new SparkSession(…) … } } Cluster Manager Spark Executor task task task task Spark Executor task task task task …
- 19. Files Sockets REST ZooKeeper Cluster ZK Mini-batch Spark Batch Spark … Low Latency Flink Ka5a Streams Akka Streams Beam Persistence S3, … HDFS DiskDiskDisk SQL/ NoSQL Search 1 5 6 KaFa Cluster Broker 2 4 7 8 9 Beam Spark Events Streams Storage Microservices Go Node.js … They do a lot (Spark example) Cluster Node input Node Node Node Time filter flatMap join map Partition 1 Partition 2 Partition 3 Partition 4 Partition 1 Partition 2 Partition 3 Partition 4 Partition 1 Partition 2 Partition 3 Partition 4 Partition 1 Partition 2 Partition 3 Partition 4 Partition 1 Partition 2 Partition 3 Partition 4 … stage1stage2 … … … …
- 20. Files Sockets REST ZooKeeper Cluster ZK Mini-batch Spark Batch Spark … Low Latency Flink Ka5a Streams Akka Streams Beam Persistence S3, … HDFS DiskDiskDisk SQL/ NoSQL Search 1 5 6 KaFa Cluster Broker 2 4 7 8 9 Beam Spark Events Streams Storage Microservices Go Node.js … Akka Streams, Kafka Streams - libraries for “data-centric microservices”. Smaller scale, but great flexibility Streaming Engines
- 21. “Record-centric” μ-services Events Records A Spectrum of Microservices Event-driven μ-services … Browse REST AccountOrders Shopping Cart API Gateway Inventory storage Data Model Training Model Serving Other Logic ← Data Spectrum →
- 22. A Spectrum of Microservices … Browse REST AccountOrders Shopping Cart API Gateway Inventory • Each datum has an identity • Process each one uniquely • Think sessions and state machines
- 23. “Record-centric” μ-services Events Records A Spectrum of Microservices Event-driven μ-services … Browse REST AccountOrders Shopping Cart API Gateway Inventory storage Data Model Training Model Serving Other Logic ← Data Spectrum →
- 24. A Spectrum of Microservices storage Data Model Training Model Serving Other Logic • “Anonymous” records • Process en masse • Think SQL queries for analytics
- 25. “Record-centric” μ-services Events Records A Spectrum of Microservices Event-driven μ-services … Browse REST AccountOrders Shopping Cart API Gateway Inventory storage Data Model Training Model Serving Other Logic ← Data Spectrum →
- 26. Events Records A Spectrum of Microservices Event-driven μ-services … Browse REST AccountOrders Shopping Cart API Gateway Inventory Akka emerged from the left-hand side of the spectrum, the world of highly Reactive microservices. Akka Streams pushes to the right, more data-centric. https://www.reactivemanifesto.org/ ← Data Spectrum →
- 27. “Record-centric” μ-services Events Records A Spectrum of Microservices storage Data Model Training Model Serving Other Logic Emerged from the right-hand side. Kafka Streams pushes to the left, supporting many event- processing scenarios. ← Data Spectrum →
- 28. Kafka Streams
- 29. • Important stream-processing semantics, e.g., • Windowing support (e.g., group by within a window) Kafka Streams 0 Time (minutes) 1 2 3 … Analysis Server 1 Server 2 accumulate 1 1 2 2 2 2 2 2 1 1 2 2 1 1 1 … Key Collect data, Then process accumulate n Event at Server n propagated to Analysis See my O’Reilly report for details.
- 30. • Important stream-processing semantics, e.g., • Distinguish between event time and processing time Kafka Streams 0 Time (minutes) 1 2 3 … Analysis Server 1 Server 2 accumulate 1 1 2 2 2 2 2 2 1 1 2 2 1 1 1 … Key Collect data, Then process accumulate n Event at Server n propagated to Analysis
- 31. • Java API • Scala API: written by Lightbend • SQL!! Kafka Streams
- 33. Data Model Training Model Serving Raw Data Model Params Scored Records val builder = new StreamsBuilderS // New Scala Wrapper API. val data = builder.stream[Array[Byte], Array[Byte]](rawDataTopic) val model = builder.stream[Array[Byte], Array[Byte]](modelTopic) val modelProcessor = new ModelProcessor val scorer = new Scorer(modelProcessor) // scorer.score(record) used model.mapValues(bytes => Model.parseBytes(bytes)) // array => record .filter((key, model) => model.valid) // Successful? .mapValues(model => ModelImpl.findModel(model)) .process(() => modelProcessor, …) // Set up actual model data.mapValues(bytes => DataRecord.parseBytes(bytes)) .filter((key, record) => record.valid) .mapValues(record => new ScoredRecord(scorer.score(record),record)) .to(scoredRecordsTopic) val streams = new KafkaStreams( builder.build, streamsConfiguration) streams.start() sys.addShutdownHook(streams.close())
- 34. val builder = new StreamsBuilderS // New Scala Wrapper API. val data = builder.stream[Array[Byte], Array[Byte]](rawDataTopic) val model = builder.stream[Array[Byte], Array[Byte]](modelTopic) val modelProcessor = new ModelProcessor val scorer = new Scorer(modelProcessor) // scorer.score(record) used model.mapValues(bytes => Model.parseBytes(bytes)) // array => record .filter((key, model) => model.valid) // Successful? .mapValues(model => ModelImpl.findModel(model)) .process(() => modelProcessor, …) // Set up actual model data.mapValues(bytes => DataRecord.parseBytes(bytes)) .filter((key, record) => record.valid) .mapValues(record => new ScoredRecord(scorer.score(record),record)) .to(scoredRecordsTopic) val streams = new KafkaStreams( builder.build, streamsConfiguration) streams.start() sys.addShutdownHook(streams.close()) Data Model Training Model Serving Raw Data Model Params Scored Records
- 35. val builder = new StreamsBuilderS // New Scala Wrapper API. val data = builder.stream[Array[Byte], Array[Byte]](rawDataTopic) val model = builder.stream[Array[Byte], Array[Byte]](modelTopic) val modelProcessor = new ModelProcessor val scorer = new Scorer(modelProcessor) // scorer.score(record) used model.mapValues(bytes => Model.parseBytes(bytes)) // array => record .filter((key, model) => model.valid) // Successful? .mapValues(model => ModelImpl.findModel(model)) .process(() => modelProcessor, …) // Set up actual model data.mapValues(bytes => DataRecord.parseBytes(bytes)) .filter((key, record) => record.valid) .mapValues(record => new ScoredRecord(scorer.score(record),record)) .to(scoredRecordsTopic) val streams = new KafkaStreams( builder.build, streamsConfiguration) streams.start() sys.addShutdownHook(streams.close()) Data Model Training Model Serving Raw Data Model Params Scored Records
- 36. val builder = new StreamsBuilderS // New Scala Wrapper API. val data = builder.stream[Array[Byte], Array[Byte]](rawDataTopic) val model = builder.stream[Array[Byte], Array[Byte]](modelTopic) val modelProcessor = new ModelProcessor val scorer = new Scorer(modelProcessor) // scorer.score(record) used model.mapValues(bytes => Model.parseBytes(bytes)) // array => record .filter((key, model) => model.valid) // Successful? .mapValues(model => ModelImpl.findModel(model)) .process(() => modelProcessor, …) // Set up actual model data.mapValues(bytes => DataRecord.parseBytes(bytes)) .filter((key, record) => record.valid) .mapValues(record => new ScoredRecord(scorer.score(record),record)) .to(scoredRecordsTopic) val streams = new KafkaStreams( builder.build, streamsConfiguration) streams.start() sys.addShutdownHook(streams.close()) Data Model Training Model Serving Raw Data Model Params Scored Records
- 37. What’s Missing? The rest of the microservice tools you need. Embed your Kafka Streams code in microservices written with Akka…
- 38. Akka Streams
- 39. Akka Streams Event Event Event Event Event Event Event/Data Stream Consumer Consumer bounded queue back pressure back pressure back pressure • Back pressure for flow control • http://www.reactive-streams.org/
- 41. Akka Streams • Part of the Akka ecosystem • Akka Actors, Akka Cluster, Akka HTTP, Akka Persistence, … • Alpakka - rich connection library • Optimized for low overhead and latency
- 42. Akka Streams • The “gist” - calculate factorials: val source: Source[Int, NotUsed] = Source(1 to 10) val factorials = source.scan(BigInt(1)) ( (total, next) => total * next ) factorials.runWith(Sink.foreach(println)) 1 2 6 24 120 720 5040 40320 362880 3628800
- 43. val source: Source[Int, NotUsed] = Source(1 to 10) val factorials = source.scan(BigInt(1)) ( (total, next) => total * next ) factorials.runWith(Sink.foreach(println)) Akka Streams • The “gist” - calculate factorials: Source Flow Sink A “Graph” 1 2 6 24 120 720 5040 40320 362880 3628800
- 44. Akka Cluster Data Model Training Model Serving Raw Data Model Params Alpakka implicit val system = ActorSystem("ModelServing") implicit val materializer = ActorMaterializer() implicit val executionContext = system.dispatcher val modelProcessor = new ModelProcessor // Same as KS example val scorer = new Scorer(modelProcessor) // Same as KS example val modelScoringStage = new ModelScoringStage(scorer)// AS custom “stage” val dataStream: Source[Record, Consumer.Control] = Consumer.atMostOnceSource(dataConsumerSettings, Subscriptions.topics(rawDataTopic)) .map(input => DataRecord.parseBytes(input.value())) .collect{ case Success(data) => data } val modelStream: Source[ModelImpl, Consumer.Control] = Consumer.atMostOnceSource(modelConsumerSettings, Subscriptions.topics(modelTopic)) .map(input => Model.parseBytes(input.value())) .collect{ case Success(mod) => mod } .map(model => ModelImpl.findModel(model))
- 45. Akka Cluster Data Model Training Model Serving Raw Data Model Params Alpakka implicit val system = ActorSystem("ModelServing") implicit val materializer = ActorMaterializer() implicit val executionContext = system.dispatcher val modelProcessor = new ModelProcessor // Same as KS example val scorer = new Scorer(modelProcessor) // Same as KS example val modelScoringStage = new ModelScoringStage(scorer)// AS custom “stage” val dataStream: Source[Record, Consumer.Control] = Consumer.atMostOnceSource(dataConsumerSettings, Subscriptions.topics(rawDataTopic)) .map(input => DataRecord.parseBytes(input.value())) .collect{ case Success(data) => data } val modelStream: Source[ModelImpl, Consumer.Control] = Consumer.atMostOnceSource(modelConsumerSettings, Subscriptions.topics(modelTopic)) .map(input => Model.parseBytes(input.value())) .collect{ case Success(mod) => mod } .map(model => ModelImpl.findModel(model))
- 46. Akka Cluster Data Model Training Model Serving Raw Data Model Params Alpakka implicit val system = ActorSystem("ModelServing") implicit val materializer = ActorMaterializer() implicit val executionContext = system.dispatcher val modelProcessor = new ModelProcessor // Same as KS example val scorer = new Scorer(modelProcessor) // Same as KS example val modelScoringStage = new ModelScoringStage(scorer)// AS custom “stage” val dataStream: Source[Record, Consumer.Control] = Consumer.atMostOnceSource(dataConsumerSettings, Subscriptions.topics(rawDataTopic)) .map(input => DataRecord.parseBytes(input.value())) .collect{ case Success(data) => data } val modelStream: Source[ModelImpl, Consumer.Control] = Consumer.atMostOnceSource(modelConsumerSettings, Subscriptions.topics(modelTopic)) .map(input => Model.parseBytes(input.value())) .collect{ case Success(mod) => mod } .map(model => ModelImpl.findModel(model))
- 47. Akka Cluster Data Model Training Model Serving Raw Data Model Params Alpakka .collect{ case Success(data) => data } val modelStream: Source[ModelImpl, Consumer.Control] = Consumer.atMostOnceSource(modelConsumerSettings, Subscriptions.topics(modelTopic)) .map(input => Model.parseBytes(input.value())) .collect{ case Success(mod) => mod } .map(model => ModelImpl.findModel(model)) .collect{ case Success(modImpl) => modImpl } .foreach(modImpl => modelProcessor.setModel(modImpl)) modelStream.to(Sink.ignore).run() // No “sinking” required; just run dataStream .viaMat(modelScoringStage)(Keep.right) .map(result => new ProducerRecord[Array[Byte], ScoredRecord]( scoredRecordsTopic, result)) .runWith(Producer.plainSink(producerSettings))
- 48. Wrapping Up…
- 49. Free as in🍺 New second edition! lbnd.io/fast-data-book