![val consumer: KafkaConsumer[String, SomeMessage] =
createConsumer()
def pollProcessAndCommit(): Unit = {
val consumerRecords = consumer.poll(1000).asScala
consumerRecords.foreach(record => {
println(s"Record value: ${record.value.messageValue}")
})
consumer.commitAsync()
pollProcessAndCommit()
}
pollProcessAndCommit()
Kafka
Consumer API
* Broker location, serde](https://image.slidesharecdn.com/greyhound-powerfulpurefunctionalkafkalibrary-devtalksreimagined-210609063549/85/Greyhound-Powerful-Functional-Kafka-Library-Devtalks-reimagined-13-320.jpg)
![val consumer: KafkaConsumer[String, SomeMessage] =
createConsumer()
def pollProcessAndCommit(): Unit = {
val consumerRecords = consumer.poll(1000).asScala
consumerRecords.foreach(record => {
println(s"Record value: ${record.value.messageValue}")
})
consumer.commitAsync()
pollProcessAndCommit()
}
pollProcessAndCommit()
Kafka
Consumer API](https://image.slidesharecdn.com/greyhound-powerfulpurefunctionalkafkalibrary-devtalksreimagined-210609063549/85/Greyhound-Powerful-Functional-Kafka-Library-Devtalks-reimagined-14-320.jpg)
![val consumer: KafkaConsumer[String, SomeMessage] =
createConsumer()
def pollProcessAndCommit(): Unit = {
val consumerRecords = consumer.poll(1000).asScala
consumerRecords.foreach(record => {
println(s"Record value: ${record.value.messageValue}")
})
consumer.commitAsync()
pollProcessAndCommit()
}
pollProcessAndCommit()
Kafka
Consumer API](https://image.slidesharecdn.com/greyhound-powerfulpurefunctionalkafkalibrary-devtalksreimagined-210609063549/85/Greyhound-Powerful-Functional-Kafka-Library-Devtalks-reimagined-15-320.jpg)
![val handler: RecordHandler[Console, Nothing,
String, SomeMessage] =
RecordHandler { record =>
zio.console.putStrLn(record.value.messageValue)
}
GreyhoundConsumersBuilder
.withConsumer(RecordConsumer(
topic = "some-group",
group = "group-2",
handle = handler))
Greyhound
Consumer API
* No commit, wix broker location](https://image.slidesharecdn.com/greyhound-powerfulpurefunctionalkafkalibrary-devtalksreimagined-210609063549/85/Greyhound-Powerful-Functional-Kafka-Library-Devtalks-reimagined-16-320.jpg)
![trait RecordHandler[-R, +E, K, V] {
def handle(record: ConsumerRecord[K, V]): ZIO[R, E, Any]
def contramap: RecordHandler
def contramapM: RecordHandler
def mapError: RecordHandler
def withErrorHandler: RecordHandler
def ignore: RecordHandler
def provide: RecordHandler
def andThen: RecordHandler
def withDeserializers: RecordHandler
}
Composable
Handler
@NSilnitsky](https://image.slidesharecdn.com/greyhound-powerfulpurefunctionalkafkalibrary-devtalksreimagined-210609063549/85/Greyhound-Powerful-Functional-Kafka-Library-Devtalks-reimagined-19-320.jpg)
![trait RecordHandler[-R, +E, K, V] {
def handle(record: ConsumerRecord[K, V]): ZIO[R, E, Any]
def contramap: RecordHandler
def contramapM: RecordHandler
def mapError: RecordHandler
def withErrorHandler: RecordHandler
def ignore: RecordHandler
def provide: RecordHandler
def andThen: RecordHandler
def withDeserializers: RecordHandler
}
Composable
Handler
@NSilnitsky * change type](https://image.slidesharecdn.com/greyhound-powerfulpurefunctionalkafkalibrary-devtalksreimagined-210609063549/85/Greyhound-Powerful-Functional-Kafka-Library-Devtalks-reimagined-20-320.jpg)
 = {
val start = for {
// ...
handle = handler.andThen(offsets.update).handle(_)
dispatcher <- Dispatcher.make(handle, /**/)
fiber <- pollOnce(/*...*/).repeatWhile(_ == true).forkDaemon
// ...
} yield (fiber, running /*...*/)
}
def pollOnce(/*...*/) = {
// poll and handle...
@NSilnitsky
* stack safe
EventLoop
Polling](https://image.slidesharecdn.com/greyhound-powerfulpurefunctionalkafkalibrary-devtalksreimagined-210609063549/85/Greyhound-Powerful-Functional-Kafka-Library-Devtalks-reimagined-25-320.jpg)
![object EventLoop {
type Handler[-R] = RecordHandler[R, Nothing, Chunk[Byte], Chunk[Byte]]
def make[R](handler: Handler[R] /*...*/) = {
val start = for {
// ...
handle = handler.andThen(offsets.increment).handle(_)
dispatcher <- // ...
fiber <- pollOnce(/*...*/)...
// ...
} yield (fiber, running /*...*/)
}
def pollOnce(/*...*/) = {
// poll and handle...
_ <- offsets.commit
Commit Offsets
@NSilnitsky
* old ghound -> pass down to worker](https://image.slidesharecdn.com/greyhound-powerfulpurefunctionalkafkalibrary-devtalksreimagined-210609063549/85/Greyhound-Powerful-Functional-Kafka-Library-Devtalks-reimagined-26-320.jpg)
![val consumer: KafkaConsumer[String, SomeMessage] =
createConsumer()
def pollProcessAndCommit(): Unit = {
val consumerRecords = consumer.poll(1000).asScala
consumerRecords.foreach(record => {
println(s"Record value: ${record.value.messageValue}")
})
consumer.commitAsync()
pollProcessAndCommit()
}
pollProcessAndCommit()
Kafka
Consumer API](https://image.slidesharecdn.com/greyhound-powerfulpurefunctionalkafkalibrary-devtalksreimagined-210609063549/85/Greyhound-Powerful-Functional-Kafka-Library-Devtalks-reimagined-28-320.jpg)
)
Context
Propagation](https://image.slidesharecdn.com/greyhound-powerfulpurefunctionalkafkalibrary-devtalksreimagined-210609063549/85/Greyhound-Powerful-Functional-Kafka-Library-Devtalks-reimagined-54-320.jpg)
)
RecordHandler((r: ConsumerRecord[String, Contact]) => for {
context <- Context.retrieve
_ <- ContactsDB.write(r.value, context)
} yield ()
=>
RecordHandler[ContactsDB with Context, Throwable, Chunk[Byte], Chunk[Byte]]
Context
Propagation](https://image.slidesharecdn.com/greyhound-powerfulpurefunctionalkafkalibrary-devtalksreimagined-210609063549/85/Greyhound-Powerful-Functional-Kafka-Library-Devtalks-reimagined-55-320.jpg)
Greyhound - Powerful Functional Kafka Library - Devtalks reimagined
- 1. Greyhound - A Powerful Functional Kafka library Natan Silnitsky Backend Infra Developer, Wix.com [email protected] twitter@NSilnitsky linkedin/natansilnitsky natansil.com
- 2. A Scala/Java high-level SDK for Apache Kafka. Powered by ZIO, Open-Sourced Greyhound * features...
- 3. But first… a few Kafka terms
- 5. @NSilnitsky Kafka Producer Topic Partition Partition Partition Kafka Broker Topic Topic Partition Partition Partition Partition Partition Partition A few Kafka terms
- 6. @NSilnitsky Topic Topic Topic Partition Partition Partition Partition Partition Partition Partition Partition Partition Kafka Producer Partition 0 1 2 3 4 5 append-only log A few Kafka terms
- 7. @NSilnitsky Topic Topic Topic Partition Partition Partition Partition Partition Partition Partition Partition Partition Kafka Consumers A few Kafka terms 0 1 2 3 4 5 6 7 8 9 1 0 1 1 1 2 1 3 1 4 1 5 1 6 1 7 1 8 1 9 2 0 Partition
- 8. Greyhound Wraps Kafka Kafka Broker Service A Service B Kafka Consumer Kafka Producer
- 9. @NSilnitsky Kafka Broker Service A Service B Abstract so that it is easy to change for everyone Simplify APIs, with additional features Greyhound wraps Kafka Kafka Consumer Kafka Producer
- 10. @NSilnitsky Multiple APIs For Java, Scala and Wix Devs Greyhound wraps Kafka Scala Future ZIO Java Kafka Consumer Kafka Producer Kafka Broker ZIO Core Service A Service B * all logic
- 11. @NSilnitsky Greyhound wraps Kafka Scala Future ZIO Java Kafka Consumer Kafka Producer Kafka Broker ZIO Core Wix Interop OSS Private Service A Service B * defaults
- 12. Kafka Broker Service A Service B Kafka Consumer Kafka Producer - Boilerplate Greyhound wraps Kafka What do we want it to do?
- 13. val consumer: KafkaConsumer[String, SomeMessage] = createConsumer() def pollProcessAndCommit(): Unit = { val consumerRecords = consumer.poll(1000).asScala consumerRecords.foreach(record => { println(s"Record value: ${record.value.messageValue}") }) consumer.commitAsync() pollProcessAndCommit() } pollProcessAndCommit() Kafka Consumer API * Broker location, serde
- 14. val consumer: KafkaConsumer[String, SomeMessage] = createConsumer() def pollProcessAndCommit(): Unit = { val consumerRecords = consumer.poll(1000).asScala consumerRecords.foreach(record => { println(s"Record value: ${record.value.messageValue}") }) consumer.commitAsync() pollProcessAndCommit() } pollProcessAndCommit() Kafka Consumer API
- 15. val consumer: KafkaConsumer[String, SomeMessage] = createConsumer() def pollProcessAndCommit(): Unit = { val consumerRecords = consumer.poll(1000).asScala consumerRecords.foreach(record => { println(s"Record value: ${record.value.messageValue}") }) consumer.commitAsync() pollProcessAndCommit() } pollProcessAndCommit() Kafka Consumer API
- 16. val handler: RecordHandler[Console, Nothing, String, SomeMessage] = RecordHandler { record => zio.console.putStrLn(record.value.messageValue) } GreyhoundConsumersBuilder .withConsumer(RecordConsumer( topic = "some-group", group = "group-2", handle = handler)) Greyhound Consumer API * No commit, wix broker location
- 17. Functional Composition Greyhound wraps Kafka ✔ Simple Consumer API + Composable Record Handler What do we want it to do?
- 18. @NSilnitsky Kafka Broker Greyhound Consumer Kafka Consumer COMPOSABLE RECORD HANDLER Site Published Topic 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 Commit
- 19. trait RecordHandler[-R, +E, K, V] { def handle(record: ConsumerRecord[K, V]): ZIO[R, E, Any] def contramap: RecordHandler def contramapM: RecordHandler def mapError: RecordHandler def withErrorHandler: RecordHandler def ignore: RecordHandler def provide: RecordHandler def andThen: RecordHandler def withDeserializers: RecordHandler } Composable Handler @NSilnitsky
- 20. trait RecordHandler[-R, +E, K, V] { def handle(record: ConsumerRecord[K, V]): ZIO[R, E, Any] def contramap: RecordHandler def contramapM: RecordHandler def mapError: RecordHandler def withErrorHandler: RecordHandler def ignore: RecordHandler def provide: RecordHandler def andThen: RecordHandler def withDeserializers: RecordHandler } Composable Handler @NSilnitsky * change type
- 21. @NSilnitsky Kafka Broker Greyhound Consumer Kafka Consumer DRILL DOWN Site Published Topic 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5
- 22. @NSilnitsky Kafka Broker Kafka Consumer Site Published Topic 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 Event loop Greyhound Consumer
- 23. @NSilnitsky Kafka Broker Kafka Consumer Site Published Topic 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 Event loop Greyhound Consumer Workers Message Dispatcher
- 24. @NSilnitsky Kafka Broker Kafka Consumer Site Published Topic 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 Event loop Greyhound Consumer Workers Message Dispatcher DRILL DOWN
- 25. object EventLoop { def make[R](handler: Handler[R] /*...*/) = { val start = for { // ... handle = handler.andThen(offsets.update).handle(_) dispatcher <- Dispatcher.make(handle, /**/) fiber <- pollOnce(/*...*/).repeatWhile(_ == true).forkDaemon // ... } yield (fiber, running /*...*/) } def pollOnce(/*...*/) = { // poll and handle... @NSilnitsky * stack safe EventLoop Polling
- 26. object EventLoop { type Handler[-R] = RecordHandler[R, Nothing, Chunk[Byte], Chunk[Byte]] def make[R](handler: Handler[R] /*...*/) = { val start = for { // ... handle = handler.andThen(offsets.increment).handle(_) dispatcher <- // ... fiber <- pollOnce(/*...*/)... // ... } yield (fiber, running /*...*/) } def pollOnce(/*...*/) = { // poll and handle... _ <- offsets.commit Commit Offsets @NSilnitsky * old ghound -> pass down to worker
- 27. Greyhound wraps Kafka ✔ Simple Consumer API ✔ Composable Record Handler + Parallel Consumption! What do we want it to do?
- 28. val consumer: KafkaConsumer[String, SomeMessage] = createConsumer() def pollProcessAndCommit(): Unit = { val consumerRecords = consumer.poll(1000).asScala consumerRecords.foreach(record => { println(s"Record value: ${record.value.messageValue}") }) consumer.commitAsync() pollProcessAndCommit() } pollProcessAndCommit() Kafka Consumer API
- 29. @NSilnitsky Kafka Broker Site Published Topic 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 Greyhound Consumer ZIO FIBERS + QUEUES
- 30. @NSilnitsky Kafka Broker Kafka Consumer Site Published Topic 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 Event loop Greyhound Consumer Workers Message Dispatcher (THREAD-SAFE) PARALLEL CONSUMPTION * much better thruput
- 31. Greyhound wraps Kafka ✔ Simple Consumer API ✔ Composable Record Handler ✔ Parallel Consumption + Batch Consumption What do we want it to do? ...what about batch processing?
- 32. @NSilnitsky Kafka Broker Site Published Topic 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 Batch Consumer ZIO FIBERS
- 33. @NSilnitsky Kafka Broker Site Published Topic 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 Batch Consumer ZIO Fiber Pool (THREAD-SAFE) PARALLEL CONSUMPTION * much better thruput ZIO.foreachParN_ Kafka Consumer Event loop
- 34. object BatchEventLoop { ... private def handle(polled: Records) = { ZIO.foreachParN_(config.parallelism)(recordsByPartition(polled)) { case (tp, records) => handler.handle(ConsumerRecordBatch(tp.topic, tp.partition, records)) *> elState.offsets.update(records) } } } Commit Offsets @NSilnitsky
- 35. Greyhound wraps Kafka ✔ Simple Consumer API ✔ Composable Record Handler ✔ Parallel Consumption ✔ Batch Consumption + Retries! What do we want it to do? ...what about Error handling?
- 36. val retryConfig = RetryConfig.nonBlocking( 1.second, 10.minutes) GreyhoundConsumersBuilder .withConsumer(GreyhoundConsumer( topic = "some-group", group = "group-2", handle = handler, retryConfig = retryConfig)) Non-blocking Retries
- 37. @NSilnitsky Kafka Broker renew-sub-topic 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 Greyhound Consumer FAILED PROCESSING Kafka Consumer
- 38. @NSilnitsky Kafka Broker renew-sub-topic 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 renew-sub-topic-retry-0 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 renew-sub-topic-retry-1 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 Inspired by Uber RETRY! Greyhound Consumer Kafka Consumer RETRY PRODUCER
- 39. @NSilnitsky Kafka Broker renew-sub-topic 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 renew-sub-topic-retry-0 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 renew-sub-topic-retry-1 Inspired by Uber RETRY! 0 1 2 3 4 5 Greyhound Consumer Kafka Consumer RETRY PRODUCER
- 40. Retries same message on failure BLOCKING POLICY HANDLER Kafka Broker source-control- update-topic 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 Greyhound Consumer Kafka Consumer build-log-service
- 41. val retryConfig = RetryConfig.finiteBlocking( 1.second, 1.minutes) GreyhoundConsumersBuilder .withConsumer(GreyhoundConsumer( topic = "some-group", group = "group-2", handle = handler, retryConfig = retryConfig)) Blocking Retries * exponential, ZIO schedule
- 42. Retries same message on failure * lag BLOCKING POLICY HANDLER Kafka Broker source-control- update-topic 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 0 1 2 3 4 5 Greyhound Consumer Kafka Consumer build-log-service
- 44. Greyhound wraps Kafka ✔ Simple Consumer API ✔ Composable Record Handler ✔ Parallel Consumption ✔ Batch Consumption ✔ Retries + Resilient Producer What do we want it to do? and when Kafka brokers are unavailable...
- 45. + Retry on Error Kafka Broker Producer Use Case: Guarantee completion Consumer Wix Payments Service Subscription renewal Job Scheduler
- 46. + Retry on Error Kafka Broker Producer Use Case: Guarantee completion Consumer Wix Payments Service Job Scheduler
- 48. Kafka Broker Producer Wix Payments Service Job Scheduler FAILS TO PRODUCE Save message to disk
- 49. Kafka Broker Producer Wix Payments Service Job Scheduler FAILS TO PRODUCE Save message to disk Retry on failure
- 50. Greyhound wraps Kafka ✔ Simple Consumer API ✔ Composable Record Handler ✔ Parallel & Batch Consumption ✔ Retries ✔ Resilient Producer + Context Propagation What do we want it to do? Super cool for us
- 51. @NSilnitsky CONTEXT PROPAGATION Language User Type USER REQUEST METADATA Sign up Site-Members Service Geo ... Browser
- 53. CONTEXT PROPAGATION Site-Members Service Kafka Broker Producer Contacts Service Consumer Browser
- 54. context = contextFrom(record.headers, token) handler.handle(record).provideSomeLayer[UserEnv](Context.layerFrom(context)) Context Propagation
- 55. context = contextFrom(record.headers, token) handler.handle(record, controller).provideSomeLayer[UserEnv](Context.layerFrom(context)) RecordHandler((r: ConsumerRecord[String, Contact]) => for { context <- Context.retrieve _ <- ContactsDB.write(r.value, context) } yield () => RecordHandler[ContactsDB with Context, Throwable, Chunk[Byte], Chunk[Byte]] Context Propagation
- 56. Greyhound wraps Kafka more features ✔ Simple Consumer API ✔ Composable Record Handler ✔ Parallel & Batch Consumption ✔ Retries ✔ Resilient Producer ✔ Context Propagation ✔ Pause/resume consumption ✔ Metrics reporting
- 57. Greyhound wraps Kafka ✔ Simple Consumer API ✔ Composable Record Handler ✔ Parallel Consumption ✔ Retries ✔ Resilient Producer ✔ Context Propagation ✔ Pause/resume consumption ✔ Metrics reporting future plans + Exactly Once Processing + Other Consumer strategies + Greyhound as a Service Will be much simpler with ZIO
- 59. GREYHOUND USE CASES AT WIX Pub/Sub CDC Offline Scheduled Jobs DB (Elastic Search) replication Action retries Materialized Views
- 60. - Much less boilerplate - Code that’s easier to understand - Fun - Introduce new features faster REWRITING GREYHOUND IN ZIO RESULTED IN...
- 61. A Scala/Java high-level SDK for Apache Kafka. 0.2 is out! github.com/wix/greyhound
- 62. Thank You natansil.com twitter@NSilnitsky linkedin/natansilnitsky github.com/natansil