docker-compose部署kafka集群

背景

用于测试、开发环境部署kafka集群，实际上也可以用于生产环境，但是需要调整kafka集群参数、镜像系统参数，以达到kafka运行的最佳环境。

本例因为资源有限，在一台服务器上进行集群模拟安装。

安装zookeeper集群

创建工作目录

mkdir /data/zk

编写docker-compose文件

version:'3'services:zoo1:image: zookeeper
    restart: always
    hostname: zoo1
    container_name: zoo1
    # 每台zk对外的端口ports:- 12181:2181volumes:# 数据目录及快照目录- /data/zk/zk1/data:/data
      - /data/zk/zk1/datalog:/datalog
    environment:# zk服务器唯一IdZOO_MY_ID:1# zk集群的服务器ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181

  zoo2:image: zookeeper
    restart: always
    hostname: zoo2
    container_name: zoo2
    ports:- 22181:2181volumes:- /data/zk/zk2/data:/data
      - /data/zk/zk2/datalog:/datalog
    environment:ZOO_MY_ID:2ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181

  zoo3:image: zookeeper
    restart: always
    hostname: zoo3
    container_name: zoo3
    ports:- 32181:2181volumes:- /data/zk/zk3/data:/data
      - /data/zk/zk3/datalog:/datalog
    environment:ZOO_MY_ID:3ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181

启动zk集群

cd /data/zk
docker-compose up -d

安装kafka集群

创建工作目录

mkdir /data/kafka

创建yaml文件

version:'3'services:kafka1:image: wurstmeister/kafka
    container_name: kafka1
    ports:-"19092:9092"-"19999:19999"environment:#镜像的参数规则参考：https://github.com/wurstmeister/kafka-dockerKAFKA_BROKER_ID:0KAFKA_NUM_PARTITIONS:3KAFKA_DEFAULT_REPLICATION_FACTOR:2KAFKA_ZOOKEEPER_CONNECT: 10.10.8.23:12181,10.10.8.23:22181,10.10.8.23:32181/devkafka
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://10.10.8.23:19092JMX_PORT:19999KAFKA_JMX_OPTS:"-Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.rmi.port=19999 -Dcom.sun.management.jmxremote.port=19999 -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false"volumes:- /data/kafka/broker1/logs:/opt/kafka/logs
    restart: always
 
  kafka2:image: wurstmeister/kafka
    container_name: kafka2
    ports:-"29092:9092"-"29999:29999"environment:KAFKA_BROKER_ID:1KAFKA_NUM_PARTITIONS:3KAFKA_DEFAULT_REPLICATION_FACTOR:2KAFKA_ZOOKEEPER_CONNECT: 10.10.8.23:12181,10.10.8.23:22181,10.10.8.23:32181/devkafka
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://10.10.8.23:29092JMX_PORT:29999KAFKA_JMX_OPTS:"-Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.rmi.port=29999 -Dcom.sun.management.jmxremote.port=29999 -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false"volumes:- /data/kafka/broker2/logs:/opt/kafka/logs
      - /var/run/docker.sock:/var/run/docker.sock
    restart: always
  
  kafka3:image: wurstmeister/kafka
    container_name: kafka3
    ports:-"39092:9092"-"39999:39999"environment:KAFKA_BROKER_ID:2KAFKA_NUM_PARTITIONS:3KAFKA_DEFAULT_REPLICATION_FACTOR:2KAFKA_ZOOKEEPER_CONNECT: 10.10.8.23:2182,10.10.8.23:22181,10.10.8.23:32181/devkafka
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://10.10.8.23:39092JMX_PORT:39999KAFKA_JMX_OPTS:"-Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.rmi.port=39999 -Dcom.sun.management.jmxremote.port=39999 -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false"volumes:- /data/kafka/broker3/logs:/opt/kafka/logs
      - /var/run/docker.sock:/var/run/docker.sock
    restart: always

启动集群

cd /data/kafka
docker-compose up -d