文章目录
一、flume配置
1、flume安装配置
1.1 flume下载
mkdir -p /DATA/disk1/software/flume/;
cd /DATA/disk1/software/flume/;
wget http://archive.apache.org/dist/flume/1.9.0/apache-flume-1.9.0-bin.tar.gz;
[root@cdh-client-v01 ~]# mkdir -p /DATA/disk1/software/flume/[root@cdh-client-v01 ~]# cd /DATA/disk1/software/flume[root@cdh-client-v01 flume]# wget http://archive.apache.org/dist/flume/1.9.0/apache-flume-1.9.0-bin.tar.gz--2022-10-06 21:16:51-- http://archive.apache.org/dist/flume/1.9.0/apache-flume-1.9.0-bin.tar.gz
Resolving archive.apache.org (archive.apache.org)... 138.201.131.134, 2a01:4f8:172:2ec5::2
Connecting to archive.apache.org (archive.apache.org)|138.201.131.134|:80... connected.
HTTP request sent, awaiting response... 200 OK
Length: 67938106 (65M)[application/x-gzip]
Saving to: ‘apache-flume-1.9.0-bin.tar.gz’
100%[=========================================================================================================================================================================================>] 67,938,106 1.83MB/s in 51s
2022-10-06 21:17:42 (1.27 MB/s)- ‘apache-flume-1.9.0-bin.tar.gz’ saved [67938106/67938106]
1.2 flume解压缩
cd /DATA/disk1/software/flume/;
mkdir -p /DATA/disk1/apps/;
tar -xvf apache-flume-1.9.0-bin.tar.gz -C /DATA/disk1/apps/
[root@cdh-client-v01 flume]# mkdir -p /DATA/disk1/apps[root@cdh-client-v01 flume]# tar -xvf apache-flume-1.9.0-bin.tar.gz -C /DATA/disk1/apps/
apache-flume-1.9.0-bin/conf/
apache-flume-1.9.0-bin/bin/
apache-flume-1.9.0-bin/doap_Flume.rdf
apache-flume-1.9.0-bin/README.md
apache-flume-1.9.0-bin/DEVNOTES
apache-flume-1.9.0-bin/conf/log4j.properties
apache-flume-1.9.0-bin/conf/flume-env.sh.template
apache-flume-1.9.0-bin/conf/flume-conf.properties.template
apache-flume-1.9.0-bin/conf/flume-env.ps1.template
******
apache-flume-1.9.0-bin/lib/flume-taildir-source-1.9.0.jar
apache-flume-1.9.0-bin/lib/flume-ng-environment-variable-config-filter-1.9.0.jar
apache-flume-1.9.0-bin/lib/flume-ng-hadoop-credential-store-config-filter-1.9.0.jar
apache-flume-1.9.0-bin/lib/flume-ng-external-process-config-filter-1.9.0.jar
apache-flume-1.9.0-bin/lib/flume-ng-log4jappender-1.9.0.jar
apache-flume-1.9.0-bin/lib/flume-tools-1.9.0.jar
apache-flume-1.9.0-bin/lib/slf4j-log4j12-1.7.25.jar
apache-flume-1.9.0-bin/lib/apache-log4j-extras-1.1.jar
apache-flume-1.9.0-bin/tools/flume-ng-log4jappender-1.9.0-jar-with-dependencies.jar
1.3 flume软连接
cd /DATA/disk1/apps/;
chown -R flumea:flumea ./apache-flume-1.9.0-bin;
ln -s ./apache-flume-1.9.0-bin ./flume;
[root@cdh-client-v01 apps]# cd /DATA/disk1/apps/[root@cdh-client-v01 apps]# chown -R flumea:flumea ./apache-flume-1.9.0-bin[root@cdh-client-v01 apps]# ln -s ./apache-flume-1.9.0-bin ./flume[root@cdh-client-v01 apps]# ll
total 4
drwxr-xr-x 7 flumea flumea 4096 Oct 6 21:18 apache-flume-1.9.0-bin
lrwxrwxrwx 1 root root 24 Oct 6 21:56 flume -> ./apache-flume-1.9.0-bin
1.4 flume生成日志到指定文件
1.4.1 sar命令安装
yum -y install sysstat
[root@cdh-client-v01 apps]# yum -y install sysstat
Loaded plugins: fastestmirror
Determining fastest mirrors
Could not get metalink https://mirrors.fedoraproject.org/metalink?repo=epel-7&arch=x86_64 error was
12: Timeout on https://mirrors.fedoraproject.org/metalink?repo=epel-7&arch=x86_64: (28,'Operation timed out after 30002 milliseconds with 0 out of 0 bytes received')* base: mirrors.bupt.edu.cn
* epel: mirror.misakamikoto.network
* extras: mirrors.bupt.edu.cn
* updates: mirrors.bupt.edu.cn
base | 3.6 kB 00:00:00
cloudera-manager | 2.9 kB 00:00:00
extras | 2.9 kB 00:00:00
updates | 2.9 kB 00:00:00
(1/2): extras/7/x86_64/primary_db | 249 kB 00:00:00
(2/2): updates/7/x86_64/primary_db | 17 MB 00:00:01
Resolving Dependencies
--> Running transaction check
---> Package sysstat.x86_64 0:10.1.5-19.el7 will be installed
--> Processing Dependency: libsensors.so.4()(64bit)for package: sysstat-10.1.5-19.el7.x86_64
--> Running transaction check
---> Package lm_sensors-libs.x86_64 0:3.4.0-8.20160601gitf9185e5.el7 will be installed
--> Finished Dependency Resolution
Dependencies Resolved
===================================================================================================================================================================================================================================
Package Arch Version Repository Size
===================================================================================================================================================================================================================================
Installing:
sysstat x86_64 10.1.5-19.el7 base 315 k
Installing for dependencies:
lm_sensors-libs x86_64 3.4.0-8.20160601gitf9185e5.el7 base 42 k
Transaction Summary
===================================================================================================================================================================================================================================
Install 1 Package (+1 Dependent package)
Total download size: 357 k
Installed size: 1.2 M
Downloading packages:
(1/2): sysstat-10.1.5-19.el7.x86_64.rpm | 315 kB 00:00:00
(2/2): lm_sensors-libs-3.4.0-8.20160601gitf9185e5.el7.x86_64.rpm | 42 kB 00:00:00
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
Total 1.0 MB/s | 357 kB 00:00:00
Running transaction check
Running transaction test
Transaction test succeeded
Running transaction
Installing : lm_sensors-libs-3.4.0-8.20160601gitf9185e5.el7.x86_64 1/2
Installing : sysstat-10.1.5-19.el7.x86_64 2/2
Verifying : sysstat-10.1.5-19.el7.x86_64 1/2
Verifying : lm_sensors-libs-3.4.0-8.20160601gitf9185e5.el7.x86_64 2/2
Installed:
sysstat.x86_64 0:10.1.5-19.el7
Dependency Installed:
lm_sensors-libs.x86_64 0:3.4.0-8.20160601gitf9185e5.el7
Complete!
1.4.2 生成日志到指定文件
export LANG=“en_US.UTF-8”;
mkdir -p /DATA/disk1/net_logs/
cd /DATA/disk1/net_logs/;
chown -R flumea:flumea /DATA/disk1/net_logs/;
chmod 777 /DATA/disk1/net_logs/;
nohup sar -n DEV 1|grep -E “enp0s3|lo” >> /DATA/disk1/net_logs/net_logs.log &
[root@cdh-client-v01 apps]# ll /etc/sysconfig/network-scripts/ifcfg-enp0s3 -rw-r--r-- 1 root root 236 Sep 18 09:19 /etc/sysconfig/network-scripts/ifcfg-enp0s3
[root@cdh-client-v01 apps]# export LANG="en_US.UTF-8";[root@cdh-client-v01 apps]# mkdir -p /DATA/disk1/net_logs/[root@cdh-client-v01 apps]# cd /DATA/disk1/net_logs/;[root@cdh-client-v01 net_logs]# chown -R flumea:flumea /DATA/disk1/net_logs/;[root@cdh-client-v01 net_logs]# chmod 777 /DATA/disk1/net_logs/;[root@cdh-client-v01 net_logs]# nohup sar -n DEV 1|grep -E "enp0s3|lo" >> /DATA/disk1/net_logs/net_logs.log &[1] 28190
[root@cdh-client-v01 net_logs]# nohup: ignoring input and redirecting stderr to stdout
1.5 flume flume_to_kafka.conf配置文件
vi /DATA/disk1/apps/flume/conf/flume_to_kafka.conf
a1.sources = netstat_r
a1.sinks = netstat_k_kafka
a1.channels = netstat_c_kafka
a1.sources.netstat_r.type = exec
a1.sources.netstat_r.command = tail -F /DATA/disk1/net_logs/net_logs.log
a1.sinks.netstat_k_kafka.type = org.apache.flume.sink.kafka.KafkaSink
a1.sinks.netstat_k_kafka.kafka.topic = testTopic
a1.sinks.netstat_k_kafka.kafka.bootstrap.servers = cdh-datanode-v01.yunes.com:9092,cdh-datanode-v02.yunes.com:9092,cdh-datanode-v03.yunes.com:9092
a1.sinks.netstat_k_kafka.kafka.flumeBatchSize = 200
a1.sinks.netstat_k_kafka.kafka.producer.acks = 1
a1.sinks.netstat_k_kafka.kafka.linger.ms = 1
a1.sinks.netstat_k_kafka.kafka.compression.type = snappy
a1.channels.netstat_c_kafka.type = MEMORY
a1.channels.netstat_c_kafka.capacity = 10000
a1.channels.netstat_c_kafka.transactionCapacity = 1000
a1.sources.netstat_r.channels = netstat_c_kafka
a1.sinks.netstat_k_kafka.channel = netstat_c_kafka
# Name the components on this agent
a1.sources = netstat_r
a1.sinks = netstat_k_kafka
a1.channels = netstat_c_kafka
# Describe/configure the source
a1.sources.netstat_r.type = exec
a1.sources.netstat_r.command = tail -F /DATA/disk1/net_logs/net_logs.log
# Describe the netstat_k_kafka sink
a1.sinks.netstat_k_kafka.type = org.apache.flume.sink.kafka.KafkaSink
a1.sinks.netstat_k_kafka.kafka.topic = testTopic
a1.sinks.netstat_k_kafka.kafka.bootstrap.servers = cdh-datanode-v01.yunes.com:9092,cdh-datanode-v02.yunes.com:9092,cdh-datanode-v03.yunes.com:9092
a1.sinks.netstat_k_kafka.kafka.flumeBatchSize = 200
a1.sinks.netstat_k_kafka.kafka.producer.acks = 1
a1.sinks.netstat_k_kafka.kafka.linger.ms = 1
a1.sinks.netstat_k_kafka.kafka.compression.type = snappy
# Describ netstat_c_kafka channel which buffers events in memory
a1.channels.netstat_c_kafka.type = MEMORY
a1.channels.netstat_c_kafka.capacity = 10000
a1.channels.netstat_c_kafka.transactionCapacity = 1000
# Bind the source and sink to the channel
a1.sources.netstat_r.channels = netstat_c_kafka
a1.sinks.netstat_k_kafka.channel = netstat_c_kafka
1.6 flume flume-env.sh配置
cp /DATA/disk1/apps/flume/conf/flume-env.sh.template /DATA/disk1/apps/flume/conf/flume-env.sh;
vi /DATA/disk1/apps/flume/conf/flume-env.sh
export JAVA_HOME=/usr/java/jdk1.8.0_181-cloudera
export FLUME_CLASSPATH=/usr/local/flume/lib/*
export HADOOP_HOME=/opt/cloudera/parcels/CDH/lib/hadoop
export FLUME_CLASSPATH=H A D O O P H O M E / e t c / h a d o o p : {HADOOP_HOME}/etc/hadoop: HADOOPHOME/etc/hadoop:HADOOP_HOME/share/hadoop/client/*: H A D O O P H O M E / s h a r e / h a d o o p / c o m m o n / ∗ : HADOOP_HOME/share/hadoop/common/*: HADOOPHOME/share/hadoop/common/∗:HADOOP_HOME/share/hadoop/common/lib/*: H A D O O P H O M E / s h a r e / h a d o o p / h d f s / ∗ : HADOOP_HOME/share/hadoop/hdfs/*: HADOOPHOME/share/hadoop/hdfs/∗:HADOOP_HOME/share/hadoop/hdfs/lib/*: H A D O O P H O M E / s h a r e / h a d o o p / m a p r e d u c e / ∗ : HADOOP_HOME/share/hadoop/mapreduce/*: HADOOPHOME/share/hadoop/mapreduce/∗:HADOOP_HOME/share/hadoop/mapreduce/lib/*: H A D O O P H O M E / s h a r e / h a d o o p / t o o l s / l i b / ∗ : HADOOP_HOME/share/hadoop/tools/lib/*: HADOOPHOME/share/hadoop/tools/lib/∗:HADOOP_HOME/share/hadoop/yarn/*:$HADOOP_HOME/share/hadoop/yarn/lib/*:/usr/local/flume/lib/*
[root@cdh-client-v01 apps]# cp /DATA/disk1/apps/flume/conf/flume-env.sh.template /DATA/disk1/apps/flume/conf/flume-env.sh;[root@cdh-client-v01 apps]# vi /DATA/disk1/apps/flume/conf/flume-env.sh# Licensed to the Apache Software Foundation (ASF) under one# or more contributor license agreements. See the NOTICE file# distributed with this work for additional information# regarding copyright ownership. The ASF licenses this file# to you under the Apache License, Version 2.0 (the# "License"); you may not use this file except in compliance# with the License. You may obtain a copy of the License at## http://www.apache.org/licenses/LICENSE-2.0## Unless required by applicable law or agreed to in writing, software# distributed under the License is distributed on an "AS IS" BASIS,# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.# If this file is placed at FLUME_CONF_DIR/flume-env.sh, it will be sourced# during Flume startup.# Enviroment variables can be set here.# export JAVA_HOME=/usr/lib/jvm/java-8-oracle
export JAVA_HOME=/usr/java/jdk1.8.0_181-cloudera
export FLUME_CLASSPATH=/usr/local/flume/lib/*# 如果不需要 flume sink to hdfs，以下环境变量可以忽略# export HADOOP_HOME=/home/hadoop/apps/hadoop
export HADOOP_HOME=/opt/cloudera/parcels/CDH/lib/hadoop
export FLUME_CLASSPATH=${HADOOP_HOME}/etc/hadoop:$HADOOP_HOME/share/hadoop/client/*:$HADOOP_HOME/share/hadoop/common/*:$HADOOP_HOME/share/hadoop/common/lib/*:$HADOOP_HOME/share/hadoop/hdfs/*:$HADOOP_HOME/share/hadoop/hdfs/lib/*
:$HADOOP_HOME/share/hadoop/mapreduce/*:$HADOOP_HOME/share/hadoop/mapreduce/lib/*:$HADOOP_HOME/share/hadoop/tools/lib/*:$HADOOP_HOME/share/hadoop/yarn/*:$HADOOP_HOME/share/hadoop/yarn/lib/*:/usr/local/flume/lib/*
1.7 flume运行
cd /DATA/disk1/apps/flume/;
mkdir logs;
rm -rf ./logs/*;
./bin/flume-ng agent --conf conf
–conf-file ./conf/flume_to_kafka.conf
–name a1
-Dflume.root.logger=INFO,console > ./logs/flume_to_kafka.log 2>&1 &
[root@cdh-client-v01 apps]# cd /DATA/disk1/apps/flume/;[root@cdh-client-v01 flume]# mkdir logs;[root@cdh-client-v01 flume]# rm -rf ./logs/*;[root@cdh-client-v01 flume]# ./bin/flume-ng agent --conf conf \
> --conf-file ./conf/flume_to_kafka.conf \
> --name a1 \
> -Dflume.root.logger=INFO,console > ./logs/flume_to_kafka.log 2>&1 &
[2] 30569
1.8 flume实验
ps -ef|grep flume
[root@cdh-client-v01 flume]# ps -ef|grep flume
root 30569 16202 5 22:32 pts/0 00:00:01 /usr/java/jdk1.8.0_181-cloudera/bin/java -Xmx20m -Dflume.root.logger=INFO,console -cp/DATA/disk1/apps/flume/conf:/DATA/disk1/apps/flume/lib/*:/opt/cloudera/parcels/CDH/lib/hadoop/etc/hadoop:/opt/cloudera/parcels/CDH/lib/hadoop/share/hadoop/client/*:/opt/cloudera/parcels/CDH/lib/hadoop/share/hadoop/common/*:/opt/cloudera/parcels/CDH/lib/hadoop/share/hadoop/common/lib/*:/opt/cloudera/parcels/CDH/lib/hadoop/share/hadoop/hdfs/*:/opt/cloudera/parcels/CDH/lib/hadoop/share/hadoop/hdfs/lib/*:/opt/cloudera/parcels/CDH/lib/hadoop/share/hadoop/mapreduce/*:/opt/cloudera/parcels/CDH/lib/hadoop/share/hadoop/mapreduce/lib/*:/opt/cloudera/parcels/CDH/lib/hadoop/share/hadoop/tools/lib/*:/opt/cloudera/parcels/CDH/lib/hadoop/share/hadoop/yarn/*:/opt/cloudera/parcels/CDH/lib/hadoop/share/hadoop/yarn/lib/*:/usr/local/flume/lib/*:/etc/hadoop/conf:/opt/cloudera/parcels/CDH-6.3.2-1.cdh6.3.2.p0.1605554/lib/hadoop/libexec/../../hadoop/lib/*:/opt/cloudera/parcels/CDH-6.3.2-1.cdh6.3.2.p0.1605554/lib/hadoop/libexec/../../hadoop/.//*:/opt/cloudera/parcels/CDH-6.3.2-1.cdh6.3.2.p0.1605554/lib/hadoop/libexec/../../hadoop-hdfs/./:/opt/cloudera/parcels/CDH-6.3.2-1.cdh6.3.2.p0.1605554/lib/hadoop/libexec/../../hadoop-hdfs/lib/*:/opt/cloudera/parcels/CDH-6.3.2-1.cdh6.3.2.p0.1605554/lib/hadoop/libexec/../../hadoop-hdfs/.//*:/opt/cloudera/parcels/CDH-6.3.2-1.cdh6.3.2.p0.1605554/lib/hadoop/libexec/../../hadoop-mapreduce/.//*:/opt/cloudera/parcels/CDH-6.3.2-1.cdh6.3.2.p0.1605554/lib/hadoop/libexec/../../hadoop-yarn/lib/*:/opt/cloudera/parcels/CDH-6.3.2-1.cdh6.3.2.p0.1605554/lib/hadoop/libexec/../../hadoop-yarn/.//*:/opt/cloudera/parcels/CDH-6.3.2-1.cdh6.3.2.p0.1605554/lib/hbase/bin/../conf:/usr/java/jdk1.8.0_181-cloudera/lib/tools.jar:/opt/cloudera/parcels/CDH-6.3.2-1.cdh6.3.2.p0.1605554/lib/hbase/bin/..:/opt/cloudera/parcels/CDH-6.3.2-1.cdh6.3.2.p0.1605554/lib/hbase/bin/../lib/shaded-clients/hbase-shaded-client-byo-hadoop-2.1.0-cdh6.3.2.jar:/opt/cloudera/parcels/CDH-6.3.2-1.cdh6.3.2.p0.1605554/lib/hbase/bin/../lib/client-facing-thirdparty/audience-annotations-0.5.0.jar:/opt/cloudera/parcels/CDH-6.3.2-1.cdh6.3.2.p0.1605554/lib/hbase/bin/../lib/client-facing-thirdparty/commons-logging-1.2.jar:/opt/cloudera/parcels/CDH-6.3.2-1.cdh6.3.2.p0.1605554/lib/hbase/bin/../lib/client-facing-thirdparty/findbugs-annotations-1.3.9-1.jar:/opt/cloudera/parcels/CDH-6.3.2-1.cdh6.3.2.p0.1605554/lib/hbase/bin/../lib/client-facing-thirdparty/htrace-core4-4.2.0-incubating.jar:/opt/cloudera/parcels/CDH-6.3.2-1.cdh6.3.2.p0.1605554/lib/hbase/bin/../lib/client-facing-thirdparty/log4j-1.2.17.jar:/opt/cloudera/parcels/CDH-6.3.2-1.cdh6.3.2.p0.1605554/lib/hbase/bin/../lib/client-facing-thirdparty/slf4j-api-1.7.25.jar:/etc/hadoop/conf:/opt/cloudera/parcels/CDH-6.3.2-1.cdh6.3.2.p0.1605554/lib/hadoop/libexec/../../hadoop/lib/*:/opt/cloudera/parcels/CDH-6.3.2-1.cdh6.3.2.p0.1605554/lib/hadoop/libexec/../../hadoop/.//*:/opt/cloudera/parcels/CDH-6.3.2-1.cdh6.3.2.p0.1605554/lib/hadoop/libexec/../../hadoop-hdfs/./:/opt/cloudera/parcels/CDH-6.3.2-1.cdh6.3.2.p0.1605554/lib/hadoop/libexec/../../hadoop-hdfs/lib/*:/opt/cloudera/parcels/CDH-6.3.2-1.cdh6.3.2.p0.1605554/lib/hadoop/libexec/../../hadoop-hdfs/.//*:/opt/cloudera/parcels/CDH-6.3.2-1.cdh6.3.2.p0.1605554/lib/hadoop/libexec/../../hadoop-mapreduce/.//*:/opt/cloudera/parcels/CDH-6.3.2-1.cdh6.3.2.p0.1605554/lib/hadoop/libexec/../../hadoop-yarn/lib/*:/opt/cloudera/parcels/CDH-6.3.2-1.cdh6.3.2.p0.1605554/lib/hadoop/libexec/../../hadoop-yarn/.//*:/etc/hadoop/conf:/opt/cloudera/parcels/CDH/lib/hadoop/*:/opt/cloudera/parcels/CDH/lib/hadoop/lib/*:/opt/cloudera/parcels/CDH-6.3.2-1.cdh6.3.2.p0.1605554/bin/../lib/zookeeper/*:/opt/cloudera/parcels/CDH-6.3.2-1.cdh6.3.2.p0.1605554/bin/../lib/zookeeper/lib/*::/conf:/lib/*-Djava.library.path=:/opt/cloudera/parcels/CDH/lib/hadoop/lib/native:/opt/cloudera/parcels/CDH/lib/hadoop/lib/native:/opt/cloudera/parcels/CDH-6.3.2-1.cdh6.3.2.p0.1605554/lib/hbase/bin/../lib/native/Linux-amd64-64 org.apache.flume.node.Application --conf-file ./conf/flume_to_kafka.conf --name a1
root 31026 16202 0 22:32 pts/0 00:00:00 grep --color=auto flume
本文转载自: https://blog.csdn.net/walykyy/article/details/127185506
版权归原作者 与自己作战 所有， 如有侵权，请联系我们删除。
版权归原作者 与自己作战 所有， 如有侵权，请联系我们删除。