This document provides a code overview of a reactive Kafka consumer and producer using Reactor Kafka and Spring Kafka dependencies. It includes methods for producing and consuming messages, processing records into events, and handling offsets in a reactive manner. The code snippets demonstrate subscribing to topics, processing messages with backpressure, and using custom subscribers for specific processing requirements.

6. @KafkaListener(topics=“myTopic”)
public void listen(ConsumerRecord<?, ?> record) throws Exception {
logger.info(record.toString());
// Do Something...
}

9. Stream<Async<T>>

10. Flux<T>

11. <dependency>
<groupId>io.projectreactor.kafka</groupId>
<artifactId>reactor-kafka</artifactId>
<version>1.2.0.RELEASE</version>
</dependency>
dependencies {
compile group: ‘io.projectreactor.kafka’, name: ‘reactor-kafka’, version: ‘1.2.0.RELEASE’
}

12. public Flux<SendResult<String>> produce(Publisher<String> publisher) {
return KafkaSender.create(SENDER_OPTIONS)
.send(publisher.map(o -> SenderRecord.create(...)));
}

13. public Flux<ReceiverRecord<String, String>> consume() {
return KafkaReceiver.create(RECEIVER_OPTIONS)
.receive();
}
Consumer

14. <dependency>
<groupId>org.springframework.kafka</groupId>
<artifactId>spring-kafka</artifactId>
<version>2.3.3.RELEASE</version>
</dependency>
dependencies {
compile group: ‘org.springframework.kafka’, name: ‘spring-kafka’, version: ‘2.3.3.RELEASE’
}

15. public Flux<SendResult<Void>> produce(String topic, String message) {
return new ReactiveKafkaProducerTemplate<>(SENDER_OPTIONS)
.send(topic, message);
}

16. public Flux<ReceiverRecord<String, String>> consume() {
return new ReactiveKafkaConsumerTemplate<>(RECEIVER_OPTIONS)
.receive();
}

27. public void process() {
consume().flatMap(this::recordToEventObject)
.flatMap(this::saveEvent)
.flatMap(this::getReceivers)
.flatMap(this::notify)
.flatMap(this::saveResult)
.subscribe();
}

34. public void process() {
consume().flatMap(this::recordToEventObject)
.groupBy(Message::key)
.flatMap(flux -> flux.sampleFirst(Duration.ofSeconds(30))
.flatMap(this::saveEvent)
.flatMap(this::getReceivers)
.flatMap(this::notify)
.flatMap(this::saveResult)
.subscribe();
}

42. public void process() {
consume().flatMap(this::recordToNotifyObject)
.groupBy(Message::key)
.flatMap(flux -> flux.buffer(Duration.ofSeconds(30))
.flatMap(this::notify)
.flatMap(this::saveResult)
.subscribe();
}

49. !

50. subscribe()
onSubscribe()
request(n)
onNext()
onNext()
onNext()

51. .subscribe(this::doSomething);

52. .subscribe(this::doSomething);
reactor.core.publisher.Flux :
public final Disposable subscribe(Consumer<? super T> consumer);
public abstract void subscribe(CoreSubscriber<? super T> actual);

53. .subscribe(this::doSomething);
reactor.core.publisher.LambdaSubscriber :
public final void onSubscribe(Subscription s) {
...
s.request(Long.MAX_VALUE);
...
}

54. .subscribe(this::doSomething);
org.reactivestreams.Subscription / java.util.concurrent.Flow.Subscription :
public interface Subscription {
public void request(long n);
}

55. public class CustomSubscriber extends BaseSubscriber<~> {
@Override
protected void hookOnSubscribe(Subscription subscription) {
request(1);
}
}

56. public class CustomSubscriber extends BaseSubscriber<~> {
@Override
protected void hookOnNext(ReceiverRecord<String, String> record) {
Mono.just(record)
.flatMap(/* Do Something */)
.subscribe(r -> {
record.receiverOffset().acknowledge();
request(1);
});
}
}

57. public class CustomSubscriber extends BaseSubscriber<~> {
@Override
protected void hookOnSubscribe(Subscription subscription) {
request(10);
}
}

58. kafkaManager.consume(topicName)
.subscribe(new CustomSubscriber(this::doSomething));

61. request(4);
onNext()
onNext()
onNext()
onNext()

62. request(4);
commit()
commit()
commit()
commit()

63. request(4);
onNext()
onNext()
onNext()
onNext()

64. request(4);
commit()
commit()
commit()
commit()

65. request(4);
onNext()
onNext()
onNext()
onNext()

66. public class CustomSubscriber extends BaseSubscriber<~> {
private FluxSink<~> offsetSink;
@Override
protected void hookOnNext(ReceiverRecord<String, String> record) {
Mono.just(record)
.flatMap(/* Do Something */)
.subscribe(r -> {
offsetSink.next(record);
request(1);
});
}
}

67. public class CustomSubscriber extends BaseSubscriber<~> {
private FluxSink<~> offsetSink;
public CustomSubscriber(Function<~> func) {
...
Flux.<~>create(sink -> offsetSink = sink)
.reduce(-1L, (last, r) -> last < r.offset()
? commit(r)
: last)
.subscribe();
}
}

70. @ReactiveKafkaListener(topics=“myTopic”)
public Mono<Void> listen(ReceiverRecord<?, ?> record) {
return Mono.just(record)
.flatMap(/* Do Something... */)
.then();
}

71. https://github.com/EleganceLESS/nhn-forward-2019
http://t0a.st/oLTV