kafka多个监听消费同一个消息 kafkalistener 消费多个topic_mob6454cc7b3ae8的技术博客_

link管理

链接快照平台

输入网页链接，自动生成快照
标签化管理网页链接

操作Topic

配置

@Component
public class PrividerKafkaConfig {
    @Value("${spring.kafka.bootstrap-servers}")
    private String bootStrapServer;
    @Bean
    public KafkaAdmin kafkaAdmin() {
        Map<String, Object> props = new HashMap<>();
        //配置Kafka实例的连接地址
        props.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootStrapServer);
        KafkaAdmin admin = new KafkaAdmin(props);
        return admin;
    @Bean
    public AdminClient adminClient() {
        return AdminClient.create(kafkaAdmin().getConfig());
}

Controller层

@RestController
@Slf4j
public class TopicController {
    @Autowired
    private AdminClient adminClient;
    @ApiOperation(value = "创建topic")
    @ApiImplicitParams({
           @ApiImplicitParam(name = "topicName", value = "topic名称",defaultValue = "first_top",
                    required = true, dataType = "string", paramType = "query"),
            @ApiImplicitParam(name = "partitions", value = "分区数", defaultValue = "4",
                    required = true, dataType = "int", paramType = "query"),
            @ApiImplicitParam(name = "replicationFactor", value = "副本数", defaultValue = "1",
                    required = true, dataType = "int", paramType = "query")
    @GetMapping("/createTopic")
    public String createTopic(String topicName,int partitions,int replicationFactor){
        adminClient.createTopics(Arrays.asList(new NewTopic(topicName,partitions,(short)replicationFactor)));
        return "create success";
    @ApiOperation(value = "查看所有的topic")
    @GetMapping("/findAllTopic")
    public String findAllTopic() throws ExecutionException, InterruptedException {
        ListTopicsResult result = adminClient.listTopics();
        Collection<TopicListing> list = result.listings().get();
        List<String> resultList = new ArrayList<>();
        for(TopicListing topicListing : list){
            resultList.add(topicListing.name());
        return JSON.toJSONString(resultList);
    @ApiOperation(value = "查看topic详情")
    @ApiImplicitParams({
           @ApiImplicitParam(name = "topicName", value = "topic名称",defaultValue = "first_top",
                    required = true, dataType = "string", paramType = "query")
    @GetMapping("/info")
    public String topicInfo(String topicName) throws ExecutionException, InterruptedException {
        DescribeTopicsResult result = adminClient.describeTopics(Arrays.asList(topicName));
        Map<String,String> resultMap = new HashMap<>();
        result.all().get().forEach((k,v)->{
            log.info("k: "+k+" ,v: "+v.toString());
            resultMap.put(k,v.toString());
        return JSON.toJSONString(resultMap);
    @ApiOperation(value = "删除topic")
    @ApiImplicitParams({
           @ApiImplicitParam(name = "topicName", value = "topic名称",defaultValue = "first_top",
                    required = true, dataType = "string", paramType = "query")
    @GetMapping("/delete")
    public String deleteTopic(String topicName){
        DeleteTopicsResult  result = adminClient.deleteTopics(Arrays.asList(topicName));
        return  JSON.toJSONString(result.values());
}

AdminClient常用方法还有

创建Topic：createTopics(Collection newTopics)
删除Topic：deleteTopics(Collection topics)
罗列所有Topic：listTopics()
增加分区：createPartitions(Map<String, NewPartitions> newPartitions)
查询Topic：describeTopics(Collection topicNames)
查询集群信息：describeCluster()
查询ACL信息：describeAcls(AclBindingFilter filter)
创建ACL信息：createAcls(Collection acls)
删除ACL信息：deleteAcls(Collection filters)
查询配置信息：describeConfigs(Collection resources)
修改配置信息：alterConfigs(Map<ConfigResource, Config> configs)
修改副本的日志目录：alterReplicaLogDirs(Map<TopicPartitionReplica, String> replicaAssignment)
查询节点的日志目录信息：describeLogDirs(Collection brokers)
查询副本的日志目录信息：describeReplicaLogDirs(Collection replicas)

发送消息

KafkaTemplate发送消息是采取异步方式发送的

发送消息三种方式

//发送带有时间戳的消息
template.send(topic, 0, System.currentTimeMillis(), "0", msg);
//使用ProducerRecord发送消息
ProducerRecord record = new ProducerRecord(topic, msg);
template.send(record);
//使用Message发送消息
Map map = new HashMap();
map.put(KafkaHeaders.TOPIC, topic);
map.put(KafkaHeaders.PARTITION_ID, 0);
map.put(KafkaHeaders.MESSAGE_KEY, "0");
GenericMessage message = new GenericMessage(msg,new MessageHeaders(map));
template.send(message);

消息结果回调

@Component
@Slf4j
public class KafkaSendResultHandler implements ProducerListener {
    @Override
    public void onSuccess(ProducerRecord producerRecord, RecordMetadata recordMetadata) {
        log.info("Message send success : " + producerRecord.toString());
    @Override
    public void onError(ProducerRecord producerRecord, Exception exception) {
        log.info("Message send error : " + producerRecord.toString());
}

发送同步消息

@GetMapping("/syncMsg")
public String syncMsg(@RequestParam String topic, @RequestParam String msg){
    try {
        template.send(topic, msg).get();
    } catch (InterruptedException e) {
        e.printStackTrace();
    } catch (ExecutionException e) {
        e.printStackTrace();
    return "success";
}

消费消息

Spring-Kafka中消息监听大致分为两种类型，一种是单条数据消费，一种是批量消费；

GenericMessageListener

@Bean
public KafkaMessageListenerContainer demoListenerContainer(ConsumerFactory consumerFactory) {
    ContainerProperties properties = new ContainerProperties("topic3");
    properties.setGroupId("group1");
      //批量消费
      properties.setMessageListener(new MessageListener<Integer,String>() {
            @Override
            public void onMessage(ConsumerRecord<Integer, String> record) {
                log.info("topic3: " + record.toString());
        });*/
    //批量消费
    properties.setMessageListener(
        new BatchAcknowledgingConsumerAwareMessageListener<String,String>(){
            @Override
            public void onMessage(List<ConsumerRecord<String, String>> list,
                                  Acknowledgment acknowledgment, Consumer<?, ?> consumer) {
                log.info("size:{}",list.size());
    return new KafkaMessageListenerContainer(consumerFactory, properties);
}

其它MessageListener，BatchAcknowledgingConsumerAwareMessageListener为GenericMessageListener的实现类

@KafkaListener

@Component
@Slf4j
public class KafkaConsumer {
    //单条消息
      @KafkaListener(topics = {"first_top2"})
      public void consumer(ConsumerRecord<?, ?> record){
          Optional<?> kafkaMessage = Optional.ofNullable(record.value());
          if (kafkaMessage.isPresent()) {
              Object message = kafkaMessage.get();
              log.info("record =" + record);
              log.info(" message =" + message);
      //批量消息
    @KafkaListener(topics = {"first_top"},containerFactory="batchFactory")
    public void consumerBatch(List<ConsumerRecord<?, ?>> record){
        log.info("接收到消息数量：{}",record.size());
}

@Bean
public KafkaListenerContainerFactory<?> batchFactory(ConsumerFactory consumerFactory){
    ConcurrentKafkaListenerContainerFactory<Integer,String> factory =
    new ConcurrentKafkaListenerContainerFactory<>();
    factory.setConsumerFactory(consumerFactory);
    factory.setConcurrency(10);
    factory.getContainerProperties().setPollTimeout(1500);
    factory.setBatchListener(true);//设置为批量消费，每个批次数量在Kafka配置参数中设置
    return factory;
}

application.yml

messages:
    basename: i18n/Messages,i18n/Pages
  kafka:
    #bootstrap-servers: 10.10.2.138:9092,10.10.2.138:9093,10.10.2.138:9094
    bootstrap-servers: 47.106.106.53:9092
    template:
      default-topic: self-topic0
    consumer:
          key-deserializer: org.apache.kafka.common.serialization.StringDeserializer
          value-deserializer: org.apache.kafka.common.serialization.StringDeserializer
          group-id: myGroup998
          # 最早未被消费的offset
          auto-offset-reset: earliest
          # 批量一次最大拉取数据量
          max-poll-records: 1000
          # 自动提交
          enable-auto-commit: true
    consumer-extra:
           # 是否批量处理
           batch-listener: true

@KafkaListener 属性

id：消费者的id，当GroupId没有被配置的时候，默认id为GroupId
containerFactory：上面提到了@KafkaListener区分单数据还是多数据消费只需要配置一下注解的containerFactory属性就可以了，这里面配置的是监听容器工厂，也就是ConcurrentKafkaListenerContainerFactory，配置BeanName
topics：需要监听的Topic，可监听多个
topicPartitions：可配置更加详细的监听信息，必须监听某个Topic中的指定分区，或者从offset为200的偏移量开始监听
errorHandler：监听异常处理器，配置BeanName
groupId：消费组ID
idIsGroup：id是否为GroupId
clientIdPrefix：消费者Id前缀
beanRef：真实监听容器的BeanName，需要在 BeanName前加 “__”

监听Topic中指定的分区

@KafkaListener(id = "id0", containerFactory="batchFactory",
               topicPartitions = { @TopicPartition(topic = TPOIC, partitions = { "0" }) })
public void listenPartition0(List<ConsumerRecord<?, ?>> records) {
    log.info("Id0 Listener, Thread ID: " + Thread.currentThread().getId());
    log.info("Id0 records size " +  records.size());
    for (ConsumerRecord<?, ?> record : records) {
        Optional<?> kafkaMessage = Optional.ofNullable(record.value());
        log.info("Received: " + record);
        if (kafkaMessage.isPresent()) {
            Object message = record.value();
            String topic = record.topic();
            log.info("p0 Received message={}",  message);
}

注解方式获取消息头及消息体

@KafkaListener(id = "group3", topics = "first_top3")
    public void annoListener(@Payload String data,
                             @Header(KafkaHeaders.RECEIVED_MESSAGE_KEY) String key,
                             @Header(KafkaHeaders.RECEIVED_PARTITION_ID) String partition,
                             @Header(KafkaHeaders.RECEIVED_TOPIC) String topic,
                             @Header(KafkaHeaders.RECEIVED_TIMESTAMP) String ts) {
        log.info(" receive : \n"+
                "data : "+data+"\n"+
                "key : "+key+"\n"+
                "partitionId : "+partition+"\n"+
                "topic : "+topic+"\n"+
                "timestamp : "+ts+"\n"
    }

使用Ack机制确认消费

RabbitMQ的消费可以说是一次性的，也就是你确认消费后就 立刻从硬盘或内存中删除 ，而且RabbitMQ粗糙点来说是顺序消费，像排队一样，一个个顺序消费，未被确认的消息则会重新回到队列中，等待监听器再次消费。

但Kafka不同，Kafka是通过最新保存偏移量进行消息消费的，而且确认消费的消息并不会立刻删除，所以我们可以重复的消费未被删除的数据，当第一条消息未被确认，而第二条消息被确认的时候，Kafka会保存第二条消息的偏移量，也就是说第一条消息再也不会被监听器所获取，除非是根据第一条消息的偏移量手动获取

把application.yml中的 enable-auto-commit 设置为 false ,设置为不自动提交

@Bean("ackContainerFactory")
public ConcurrentKafkaListenerContainerFactory ackContainerFactory(
    ConsumerFactory consumerFactory) {
    ConcurrentKafkaListenerContainerFactory factory = 
        new ConcurrentKafkaListenerContainerFactory();
    factory.getContainerProperties().setAckMode(ContainerProperties.AckMode.MANUAL_IMMEDIATE);
    factory.setConsumerFactory(consumerFactory);
    return factory;
}

@KafkaListener(id = "ack", topics = "ack",containerFactory = "ackContainerFactory")
public void ackListener(ConsumerRecord record, Acknowledgment ack) {
    log.info("receive : " + record.value());
    //手动提交
    //  ack.acknowledge();
}

实现消息转发

@KafkaListener(id = "forward", topics = "first_top4")
@SendTo("first_top2")
public String forward(String data) {
    log.info("接收到消息数量：{}",data);
    return "send msg : " + data;
}

启动关闭监听

@RestController
public class ConsumerContoller {
    @Autowired
    private KafkaListenerEndpointRegistry registry;
    @Autowired
    private ConsumerFactory consumerFactory;
    @GetMapping("/stop")
    public String stop(){
        registry.getListenerContainer("forward").pause();
        return "success";
    @GetMapping("/start")
    public String start(){
       //判断监听容器是否启动，未启动则将其启动
        if (!registry.getListenerContainer("forward").isRunning()) {
            registry.getListenerContainer("forward").start();
        registry.getListenerContainer("forward").resume();
        return "success";
}

启动类要添加 @EnableKafka

配置消息过滤器

消息过滤器可以在消息抵达监听容器前被拦截，过滤器根据系统业务逻辑去筛选出需要的数据再交由KafkaListener处理。

/**
     * 消息过滤
     * @return
    @Bean
public ConcurrentKafkaListenerContainerFactory filterContainerFactory(
    ConsumerFactory consumerFactory) {
    ConcurrentKafkaListenerContainerFactory factory = 
        new ConcurrentKafkaListenerContainerFactory();
    factory.setConsumerFactory(consumerFactory);
    //配合RecordFilterStrategy使用，被过滤的信息将被丢弃
    factory.setAckDiscarded(true);
    factory.setRecordFilterStrategy(new RecordFilterStrategy() {
        @Override
        public boolean filter(ConsumerRecord consumerRecord) {
            String msg = (String) consumerRecord.value();
            if(msg.contains("abc")){
                return false;
            log.info("filterContainerFactory filter : "+msg);
            //返回true将会被丢弃
            return true;
    return factory;
}

public class FilterListener {
    @KafkaListener(topics = {"filter_topic"},containerFactory="filterContainerFactory")
    public void consumerBatch(ConsumerRecord<?, ?> record){
        Optional<?> kafkaMessage = Optional.ofNullable(record.value());
        if (kafkaMessage.isPresent()) {
            Object message = kafkaMessage.get();
            log.info("record =" + record);
            log.info("接收到消息数量：{}",message);