kafka启动、状态监控、日常操作

进入kafka目录下：启动zk

./bin/zookeeper-server-start.sh -daemon config/zookeeper.properties

进入kafka目录下：启动kafka

JMX_PORT=19997 bin/kafka-server-start.sh -daemon config/server.properties &

查看启动状态三张方式： jps -ml

netstat -nalpt | grep 9092

lsof -i:9092

创建topic

bin/kafka-topics.sh --create --zookeeper lzmhdev-node1:2181,lzmhdev-node2:2181,lzmhdev-node3:2181 --replication-factor 2 --partitions 2 --topic rtb_ad_req_info

bin/kafka-topics.sh --create --zookeeper lzmhdev10:2181,lzmhdev10:2182,lzmhdev10:2183 --replication-factor 2 --partitions 2 --topic ad_rtb_spark_log_sum

备注：

--create： 创建topic

--zookeeper ： zk集群 2181端口

--replication-factor ： 副本数

--partitions ： 分区数

--topic： 主题名称

查看主题详情

bin/kafka-topics.sh --describe --zookeeper lzmhdev-node1:2181,lzmhdev-node2:2181,lzmhdev-node3:2181 --topic test1

bin/kafka-topics.sh --describe --zookeeper lzmhdev10:2181,lzmhdev10:2182,lzmhdev10:2183 --topic ad_rtb_spark_log_sum

bin/kafka-topics.sh --describe --zookeeper lzmhqa01:2181,lzmhqa01:2182,lzmhqa01:2183 --topic ad_rtb_spark_log_sum

备注：

--describe ： 详情

bin/kafka-topics.sh --describe --zookeeper lzmhdev10:2181 --topic ad_rtb_spark_log_sum

查看主题list

bin/kafka-topics.sh --list --zookeeper lzmhdev-node1:2181,lzmhdev-node2:2181,lzmhdev-node3:2181

或

bin/kafka-topics.sh --zookeeper lzmhdev-node1:2181,lzmhdev-node2:2181,lzmhdev-node3:2181 -list

bin/kafka-topics.sh --zookeeper lzmhdev10:2181,lzmhdev10:2182,lzmhdev10:2183 -list

bin/kafka-topics.sh --zookeeper lzmhqa01:2181,lzmhqa01:2182,lzmhqa01:2183 -list

删除主题

bin/kafka-topics.sh --zookeeper lzmhdev-node1:2181,lzmhdev-node2:2181,lzmhdev-node3:2181 --delete --topic test

bin/kafka-topics.sh --zookeeper lzmhdev-node1:2181 --delete --topic sparktest001

bin/kafka-topics.sh --zookeeper lzmhdev10:2181,lzmhdev10:2182,lzmhdev10:2183 --delete --topic ad_rtb_ad_report_info

bin/kafka-topics.sh --zookeeper lzmhqa01:2181,lzmhqa01:2182,lzmhqa01:2183 --delete --topic ad_rtb_ad_report_info

查看消费组list

bin/kafka-consumer-groups.sh --bootstrap-server lzmhdev10:9092,lzmhdev09:9092 --list

bin/kafka-consumer-groups.sh --bootstrap-server lzmhqa01:9092 --list

查看消费组信息

bin/kafka-consumer-groups.sh --describe --bootstrap-server lzmhdev10:9092 --group adSparkLogCountgDspGroupId

bin/kafka-consumer-groups.sh --describe --bootstrap-server lzmhdev10:9092 --group adDataPersistGroupId

bin/kafka-consumer-groups.sh --describe --bootstrap-server lzmhqa01:9092 --group adDataPersistGroupId

删除消费组

bin/kafka-consumer-groups.sh --bootstrap-server lzmhdev10:9092,lzmhdev09:9092 --delete --group adDataPersistGroupId

bin/kafka-consumer-groups.sh --bootstrap-server lzmhqa01:9092 --delete --group adSparkLogCountGroupId

设置为最初偏移量

bin/kafka-consumer-groups.sh --bootstrap-server lzmhdev10:9092,lzmhdev09:9092 --group adDataPersistGroupId --topic ad_rtb_spark_log_sum --reset-offsets --to-earliest -execute

bin/kafka-consumer-groups.sh --bootstrap-server lzmhqa01:9092 --group adSparkLogCountGroupId --topic rtb_ad_req_info --reset-offsets --to-earliest -execute

设置任意偏移量

bin/kafka-consumer-groups.sh --bootstrap-server lzmhdev10:9092,lzmhdev09:9092 --group adDataPersistGroupId --topic ad_rtb_spark_log_sum --reset-offsets --to-offset 4 -execute

bin/kafka-consumer-groups.sh --bootstrap-server lzmhqa01:9092 --group adSparkLogCountGroupId --topic rtb_ad_req_info --reset-offsets --to-offset 5210857 -execute

bin/kafka-consumer-groups.sh --bootstrap-server lzmhqa01:9092 --group lzctest --topic rtb_ad_req_info --reset-offsets --to-offset 13573951 -execute

bin/kafka-consumer-groups.sh --bootstrap-server lzmhqa01:9092 --group lzctest --topic rtb_ad_rep_info --reset-offsets --to-offset 21728285 -execute

创建生产者：

bin/kafka-console-producer.sh --broker-list lzmhdev-node1:9092,lzmhdev-node2:9092,lzmhdev-node3:9092 --topic test2

bin/kafka-console-producer.sh --broker-list PLAINTEXT://lzmhdev-node3:9092 --topic test2

bin/kafka-console-producer.sh --broker-list lzmhdev-node1:9092 --topic sparktest001

bin/kafka-console-producer.sh --broker-list lzmhdev10:9092,lzmhdev09:9092 --topic ad_rtb_spark_log_sum

bin/kafka-console-producer.sh --broker-list lzmhqa01:9092 --topic rtb_ad_req_info

创建消费者

bin/kafka-console-consumer.sh --bootstrap-server lzmhdev-node1:9092,lzmhdev-node2:9092,lzmhdev-node3:9092 --topic test2 --from-beginning --consumer-property group.id=test-group

bin/kafka-console-consumer.sh --bootstrap-server lzmhdev-node1:9092,lzmhdev-node2:9092,lzmhdev-node3:9092 --topic input --from-beginning

bin/kafka-console-consumer.sh --bootstrap-server lzmhdev-node1:9092,lzmhdev-node2:9092,lzmhdev-node3:9092 --topic pTest2 --from-beginning

bin/kafka-console-consumer.sh --bootstrap-server PLAINTEXT://lzmhdev-node3:9092 --topic test2 --from-beginning

bin/kafka-console-consumer.sh --bootstrap-server lzmhdev-node1:9092 --topic sparksink001 --from-beginning

bin/kafka-console-consumer.sh --bootstrap-server lzmhdev10:9092 --topic ad_rtb_spark_log_sum --from-beginning --consumer-property group.id=ad_rtb_log

bin/kafka-console-consumer.sh --bootstrap-server lzmhdev10:9092,lzmhdev09:9092 --topic ad_rtb_spark_log_sum --from-beginning --consumer-property group.id=ad_rtb_log

bin/kafka-console-consumer.sh --bootstrap-server lzmhdev10:9092,lzmhdev09:9092 --topic ad_rtb_spark_log_sum --from-beginning --consumer-property group.id=lzctest

bin/kafka-console-consumer.sh --bootstrap-server lzmhqa01:9092 --topic rtb_ad_rep_info --from-beginning --consumer-property group.id=lzctest