kafka3.7 集群 docker部署 springboot

前提

你需要知道docker使用、springboot整合kafka

拉取kafka3.7镜像

docker pull apache/kafka:3.7.0

创建数据卷映射

创建以下两个目录（演示使用两个kafka节点）用于日志存储

/logs/kafka/kafka-node1
/logs/kafka/kafka-node2
#设置文件权限
sudo chmod 777 /logs/kafka/*

编写kafka-compose.yml

version: '3'

services:
  kafka-node1:
    image: apache/kafka:3.7.0
    container_name: kafka-node1
    networks:
      - kafka-cluster
    ports:
      - "9191:9092"
      - "9192:9093"
    volumes:
      - /logs/kafka/kafka-node1:/tmp/kafka-logs
    environment:
      KAFKA_PROCESS_ROLES: "broker,controller"
      KAFKA_NODE_ID: "1"
      KAFKA_CONTROLLER_QUORUM_VOTERS: "1@kafka-node1:9093,2@kafka-node2:9093"
      KAFKA_LISTENERS: "PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093"
      KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://192.168.86.99:9191"
      KAFKA_CONTROLLER_LISTENER_NAMES: "CONTROLLER"
      KAFKA_INTER_BROKER_LISTENER_NAME: "PLAINTEXT"
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "1"
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: "1"
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: "1"

  kafka-node2:
    image: apache/kafka:3.7.0
    container_name: kafka-node2
    networks:
      - kafka-cluster
    ports:
      - "9193:9092"
      - "9194:9093"
    volumes:
      - /logs/kafka/kafka-node2:/tmp/kafka-logs
    environment:
      KAFKA_PROCESS_ROLES: "broker,controller"
      KAFKA_NODE_ID: "2"
      KAFKA_CONTROLLER_QUORUM_VOTERS: "1@kafka-node1:9093,2@kafka-node2:9093"
      KAFKA_LISTENERS: "PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093"
      KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://192.168.86.99:9193"
      KAFKA_CONTROLLER_LISTENER_NAMES: "CONTROLLER"
      KAFKA_INTER_BROKER_LISTENER_NAME: "PLAINTEXT"
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "1"
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: "1"
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: "1"

networks:
  kafka-cluster:
    driver: bridge

volumes:
  kraft-data1:
    driver: local
  kraft-data2:
    driver: local

运行

docker-compose -f kafka-compose.yml up -d

docker network ls
docker network inspect kafka_kafka-cluster

会自动创建网络

使用springboot测试

创建一个topic，副本为2个。

@Configuration
public class KafkaConfig {

    @Bean
    public NewTopic newTopic() {
        return new NewTopic("topic-consumer-partition", 10, (short) 2);
    }
}

向topic发送信息，查看结果，两个节点都有备份。