docker-compose搭建zookeeper+kafka集群

一、准备好docker环境，下载zookeeper和kafka镜像

docker search zookeeper #检索zookeeper镜像
docker pull <imagename> #根据检索的结果下载自己想要的镜像和版本
docker search kafka #检索Kafka镜像
docker pull <imagename> #根据检索的结果下载自己想要的镜像和版本
docker images -a 查看已下载的镜像

二、 规划集群

1. 规划一个集群使用的子网

docker network create kafka --subnet 172.18.0.0/16 #创建一个用于kafka集群的网桥
docker network ls 查看docker的网络
docker network inspect kafka 查看网络网桥详情

1. 规划zk和kafka的IP地址
   hostnameip addrportlistenerzk1172.18.10.112182:2181zk2172.18.10.122183:2181zk3172.18.10.132184:2181kafka1172.18.10.14内部9092:9092，外部9192:9192kafka1kafka2172.18.10.15内部9093:9093，外部9193:9193kafka2kafka3172.18.10.16内部9094:9094，外部9194:9194kafka3

   三、创建zookeeper集群

2. 创建zk.yaml文件

version: '3.4'
services:
  zk1:
    image: zookeeper
    restart: always
    hostname: zk1
    container_name: zk1
    ports:
    - 2182:2181
    - 8081:8080
    environment:
        ZOO_MY_ID: 1
        ZOO_SERVERS: server.1=zk1:2888:3888;2181 server.2=zk2:2888:3888;2181 server.3=zk3:2888:3888;2181
    networks:
        kafka: 
            ipv4_address: 172.18.10.11

  zk2:   
    image: zookeeper
    restart: always
    hostname: zk2
    container_name: zk2
    ports:
    - 2183:2181
    - 8082:8080
    environment:
        ZOO_MY_ID: 2
        ZOO_SERVERS: server.1=zk1:2888:3888;2181 server.2=zk2:2888:3888;2181 server.3=zk3:2888:3888;2181
    networks:
        kafka: 
            ipv4_address: 172.18.10.12
            
  zk3:   
    image: zookeeper
    restart: always
    hostname: zk3
    container_name: zk3
    ports:
    - 2184:2181
    - 8083:8080
    environment:
        ZOO_MY_ID: 3
        ZOO_SERVERS: server.1=zk1:2888:3888;2181 server.2=zk2:2888:3888;2181 server.3=zk3:2888:3888;2181
    networks:
        kafka: 
            ipv4_address: 172.18.10.13

networks:
  kafka:
    external:
      name: kafka

docker-compose -f zk.yml up -d #创建zk集群
docker ps -a 查看创建的容器

检查zk集群是否创建成功

四、创建Kafka集群

1.创建kafka.yml文件

version: '3.4'
services:
  zk1:
    image: zookeeper
    restart: always
    hostname: zk1
    container_name: zk1
    ports:
    - 2182:2181
    - 8081:8080
    environment:
        ZOO_MY_ID: 1
        ZOO_SERVERS: server.1=zk1:2888:3888;2181 server.2=zk2:2888:3888;2181 server.3=zk3:2888:3888;2181
    networks:
        kafka: 
            ipv4_address: 172.18.10.11

  zk2:   
    image: zookeeper
    restart: always
    hostname: zk2
    container_name: zk2
    ports:
    - 2183:2181
    - 8082:8080
    environment:
        ZOO_MY_ID: 2
        ZOO_SERVERS: server.1=zk1:2888:3888;2181 server.2=zk2:2888:3888;2181 server.3=zk3:2888:3888;2181
    networks:
        kafka: 
            ipv4_address: 172.18.10.12
            
  zk3:   
    image: zookeeper
    restart: always
    hostname: zk3
    container_name: zk3
    ports:
    - 2184:2181
    - 8083:8080
    environment:
        ZOO_MY_ID: 3
        ZOO_SERVERS: server.1=zk1:2888:3888;2181 server.2=zk2:2888:3888;2181 server.3=zk3:2888:3888;2181
    networks:
        kafka: 
            ipv4_address: 172.18.10.13

networks:
  kafka:
    external:
      name: kafka

检查kafka集群是否均注册到zk上

docker exec -it 0aff4c8b6034 /bin/bash #进入到其中一台zk容器
cd bin #进入到zookeeper的bin目录
./zkCli.sh -server 127.0.0.1:2181 #进入zk的客户端
ls /brokers/ids #查看节点是否存在
get /brokers/ids/1 #查询节点信息

五、测试kafka是否正常

docker ps -a #查询容器列表
docker exec -it 8aafb38249f8 /bin/bash #进入其中的一个kafka节点
cd /opt/kafka_2.13-2.8.1/bin #切换到kafka的bin目录
kafka-topics.sh --create --zookeeper zk2:2181 --replication-factor 2 --partitions 2 --topic partopic #创建topic
kafka-topics.sh --describe --zookeeper zk1:2181 --topic partopic #检查topic是否创建成功