docker-compose 安装 es集群、kibana、cerebro、Logstash 导入测试数据

1、版本
elasticsearch:7.14.2
kibana:7.14.2
cerebro:0.9.4
logstash:7.14.0

2、文件创建与数据获取
1）es
此集群为 3 节点，故创建 es01，es02，es03 三个文件夹，在每个文件夹下新建 data、conf、plugins 文件夹。
在 es0x/conf 创建 elasticsearch.yml，添加如下内容

es01/conf/elasticsearch.yml

cluster.name: fupingstar_es_cluster # 集群名称，集群名称相同的节点自动组成一个集群node.name: es01  # 节点名称network.host: 0.0.0.0 # 同时设置bind_host和publish_hosthttp.port:9200# rest客户端连接端口transport.tcp.port:9300# 集群中节点互相通信端口node.master:true# 设置master角色node.data:true# 设置data角色node.ingest:true# 设置ingest角色 在索引之前，对文档进行预处理，支持pipeline管道，相当于过滤器bootstrap.memory_lock:falsenode.max_local_storage_nodes:1http.cors.enabled:true# 跨域配置http.cors.allow-origin: /.*/# 跨域配置

es02/conf/elasticsearch.yml

cluster.name: fupingstar_es_cluster
node.name: es02
network.host: 0.0.0.0
http.port:9201transport.tcp.port:9300node.master:truenode.data:truenode.ingest:truebootstrap.memory_lock:falsenode.max_local_storage_nodes:1http.cors.enabled:truehttp.cors.allow-origin: /.*/

es03/conf/elasticsearch.yml

cluster.name: fupingstar_es_cluster
node.name: es03
network.host: 0.0.0.0
http.port:9202transport.tcp.port:9300node.master:truenode.data:truenode.ingest:truebootstrap.memory_lock:falsenode.max_local_storage_nodes:1http.cors.enabled:truehttp.cors.allow-origin: /.*/

1. kibana 创建 kibana 文件夹，在文件夹下创建 kibana.yml 文件，在文件中添加如下内容：

注意 IP 修改成自己的

## ** THIS IS AN AUTO-GENERATED FILE **## Default Kibana configuration for docker targetserver.name: kibana
server.host:"0"elasticsearch.hosts:["http://192.168.140.131:9200"]xpack.monitoring.ui.container.elasticsearch.enabled:true

3）logstash
logstash 数据下载：https://grouplens.org/datasets/movielens/

下载后解压缩：

创建 logstash 文件夹，在其下创建 conf 文件夹和 mydata 文件夹，将解压缩出来的 movies.csv 放入 mydata 文件夹。
在 conf 文夹夹下创建两个文件，并填入以下内容：
logstash.yml

http.host:"0.0.0.0"

logstash.conf

input {
  file {
    path => "/usr/share/logstash/data/movies.csv"
    start_position => "beginning"
    sincedb_path => "/dev/null"
  }}
filter {
  csv {
    separator => ","
    columns =>["id","content","genre"]}

  mutate {
    split =>{ "genre" => "|" }
    remove_field =>["path","host","@timestamp","message"]}

  mutate {

    split =>["content","("]
    add_field =>{ "title" => "%{[content][0]}"}
    add_field =>{ "year" => "%{[content][1]}"}}

  mutate {
    convert =>{
      "year" => "integer"
    }
    strip =>["title"]
    remove_field =>["path","host","@timestamp","message","content"]}}
output {
   elasticsearch {
     hosts => "http://192.168.140.131:9200"
     index => "movies"
     document_id => "%{id}"
   }
  stdout {}}

3、编写 docker-compose.yml 文件。

修改端口、挂载目录等。

version: '3'
services: 
  es01:
    image: elasticsearch:7.14.2
    container_name: es01
    environment:
      - discovery.seed_hosts=es02,es03
      - cluster.initial_master_nodes=es01,es02,es03
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"   
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - /learn/docker/elk/es01/data:/usr/share/elasticsearch/data
      - /learn/docker/elk/es01/conf/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
      - /learn/docker/elk/es01/plugins:/usr/share/elasticsearch/plugins
    ports:
      - 9200:9200

  es02:
    image: elasticsearch:7.14.2
    container_name: es02
    environment:
      - discovery.seed_hosts=es01,es03
      - cluster.initial_master_nodes=es01,es02,es03
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - /learn/docker/elk/es02/data:/usr/share/elasticsearch/data
      - /learn/docker/elk/es02/conf/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
      - /learn/docker/elk/es02/plugins:/usr/share/elasticsearch/plugins
    ports:
      - 9201:9201

  es03:
    image: elasticsearch:7.14.2
    container_name: es03
    environment:
      - discovery.seed_hosts=es01,es02
      - cluster.initial_master_nodes=es01,es02,es03
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - /learn/docker/elk/es03/data:/usr/share/elasticsearch/data
      - /learn/docker/elk/es03/conf/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
      - /learn/docker/elk/es03/plugins:/usr/share/elasticsearch/plugins
    ports:
      - 9202:9202

  kibana:
    image: kibana:7.14.2
    container_name: kibana
    depends_on:
      - es01
    environment:
      ELASTICSEARCH_URL: http://es01:9200
      ELASTICSEARCH_HOSTS: http://es01:9200
    volumes:
      - /learn/docker/elk/kibana/kibana.yml:/usr/share/kibana/config/kibana.yml
    ports:
      - 5601:5601
  cerebro:
    image: lmenezes/cerebro:0.9.4
    container_name: cerebro
    ports:
      - "9801:9000"
    command:
      - -Dhosts.0.host=http://192.168.140.131:9200
      
   logstash:
    image: logstash:7.14.0
    container_name: elk_logstash
    restart: always
    volumes:
      - /learn/docker/elk/logstash/conf/logstash.conf:/usr/share/logstash/pipeline/logstash.conf:rw
      - /learn/docker/elk/logstash/conf/logstash.yml:/usr/share/logstash/config/logstash.yml
      - /learn/docker/elk/logstash/mydata/movies.csv:/usr/share/logstash/data/movies.csv
    depends_on:
      - es01
    ports:
      - 4560:4560

4、启动集群，在docker-compose文件所在目录下运行命令

docker-compose up -d

5、启动成功后，访问对应地址。


查看 logstash 容器日志，可看到在执行相关数据导入工作。

在 kibana 的 Dev tools 执行 GET /_cat/indices?v 查看，可以看到 movies，说明数据也导入成功。